[improvement][headless]Release brand new version of Translator module.
Some checks are pending
supersonic CentOS CI / build (21) (push) Waiting to run
supersonic mac CI / build (21) (push) Waiting to run
supersonic ubuntu CI / build (21) (push) Waiting to run
supersonic windows CI / build (21) (push) Waiting to run

This commit is contained in:
jerryjzhang
2025-01-05 00:00:18 +08:00
91 changed files with 1965 additions and 3158 deletions

View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.*;
@@ -30,6 +31,7 @@ import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
@@ -52,6 +54,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
private final DataSetService dataSetService;
private final SchemaService schemaService;
private final SemanticTranslator semanticTranslator;
private final MetricDrillDownChecker metricDrillDownChecker;
private final KnowledgeBaseService knowledgeBaseService;
private final MetricService metricService;
private final DimensionService dimensionService;
@@ -61,6 +64,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils,
SemanticSchemaManager semanticSchemaManager, DataSetService dataSetService,
SchemaService schemaService, SemanticTranslator semanticTranslator,
MetricDrillDownChecker metricDrillDownChecker,
KnowledgeBaseService knowledgeBaseService, MetricService metricService,
DimensionService dimensionService) {
this.statUtils = statUtils;
@@ -69,6 +73,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
this.dataSetService = dataSetService;
this.schemaService = schemaService;
this.semanticTranslator = semanticTranslator;
this.metricDrillDownChecker = metricDrillDownChecker;
this.knowledgeBaseService = knowledgeBaseService;
this.metricService = metricService;
this.dimensionService = dimensionService;
@@ -115,6 +120,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
semanticTranslator.translate(queryStatement);
// Check whether the dimensions of the metric drill-down are correct temporarily,
// add the abstraction of a validator later.
metricDrillDownChecker.checkQuery(queryStatement);
// 4.execute query
SemanticQueryResp queryResp = null;
for (QueryExecutor queryExecutor : queryExecutors) {
@@ -198,6 +207,14 @@ public class S2SemanticLayerService implements SemanticLayerService {
ModelResp modelResp = modelResps.get(0);
String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
modelResp.getName());
List<Dimension> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
queryDimValueReq.getDateInfo().getDateField(),
queryDimValueReq.getDateInfo().getStartDate(),
queryDimValueReq.getDateInfo().getDateField(),
queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
+ queryDimValueReq.getValue() + "%'";
@@ -269,10 +286,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
queryStatement.setIsTranslated(true);
}
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
return queryStatement;
}

View File

@@ -4,10 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;

View File

@@ -1,14 +1,16 @@
package com.tencent.supersonic.headless.server.manager;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.core.pojo.JoinRelation;
import com.tencent.supersonic.headless.core.pojo.Ontology;
import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.*;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
@@ -32,6 +34,24 @@ public class SemanticSchemaManager {
public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) {
Ontology ontology = new Ontology();
Map<String, List<MetricSchemaResp>> model2Metrics = Maps.newHashMap();
semanticSchemaResp.getMetrics().forEach(dim -> {
if (!model2Metrics.containsKey(dim.getModelBizName())) {
model2Metrics.put(dim.getModelBizName(), Lists.newArrayList());
}
model2Metrics.get(dim.getModelBizName()).add(dim);
});
ontology.setMetricMap(model2Metrics);
Map<String, List<DimSchemaResp>> model2Dimensions = Maps.newHashMap();
semanticSchemaResp.getDimensions().forEach(dim -> {
if (!model2Dimensions.containsKey(dim.getModelBizName())) {
model2Dimensions.put(dim.getModelBizName(), Lists.newArrayList());
}
model2Dimensions.get(dim.getModelBizName()).add(dim);
});
ontology.setDimensionMap(model2Dimensions);
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
@@ -45,25 +65,16 @@ public class SemanticSchemaManager {
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataModel> dataModelMap =
Map<String, ModelResp> dataModelMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect(
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
ontology.setDataModelMap(dataModelMap);
}
if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
}
ontology.setDimensionMap(dimensionMap);
}
if (!metricYamlTpls.isEmpty()) {
ontology.setMetrics(getMetrics(metricYamlTpls));
Collectors.toMap(ModelResp::getName, item -> item, (k1, k2) -> k1));
ontology.setModelMap(dataModelMap);
}
return ontology;
}
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
public static List<MetricSchemaResp> getMetrics(final List<MetricYamlTpl> t) {
return getMetricsByMetricYamlTpl(t);
}
@@ -71,36 +82,34 @@ public class SemanticSchemaManager {
return getDimension(t);
}
public static DataModel getDataModel(final DataModelYamlTpl d) {
DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
}
public static ModelResp getDataModel(final DataModelYamlTpl d) {
// ModelResp dataModel = ModelResp.builder()(d.getId()).modelId(d.getSourceId())
// .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
// .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
// .measures(getMeasureParams(d.getMeasures()))
// .dimensions(getDimensions(d.getDimensions())).build();
ModelResp dataModel = new ModelResp();
dataModel.setId(d.getId());
dataModel.setName(d.getName());
ModelDetail modelDetail = new ModelDetail();
dataModel.setModelDetail(modelDetail);
modelDetail.setDbType(d.getType());
modelDetail.setSqlQuery(d.getSqlQuery());
modelDetail.setTableQuery(d.getTableQuery());
modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
return dataModel;
}
private static String getDataModelAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
if (timeDimension.isPresent()
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
}
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
}
private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
List<Metric> metrics = new ArrayList<>();
private static List<MetricSchemaResp> getMetricsByMetricYamlTpl(
List<MetricYamlTpl> metricYamlTpls) {
List<MetricSchemaResp> metrics = new ArrayList<>();
for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {
Metric metric = new Metric();
metric.setMetricTypeParams(getMetricTypeParams(metricYamlTpl.getTypeParams()));
metric.setOwners(metricYamlTpl.getOwners());
MetricSchemaResp metric = new MetricSchemaResp();
fillMetricTypeParams(metric, metricYamlTpl.getTypeParams());
metric.setType(metricYamlTpl.getType());
metric.setName(metricYamlTpl.getName());
metrics.add(metric);
@@ -108,55 +117,50 @@ public class SemanticSchemaManager {
return metrics;
}
private static MetricTypeParams getMetricTypeParams(
private static void fillMetricTypeParams(MetricSchemaResp metric,
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
MetricTypeParams metricTypeParams = new MetricTypeParams();
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
metricTypeParams.setFieldMetric(false);
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
MetricDefineByMeasureParams params = new MetricDefineByMeasureParams();
params.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
metric.setMetricDefinition(MetricDefineType.MEASURE, params);
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
MetricDefineByMetricParams params = new MetricDefineByMetricParams();
params.setMetrics(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
params.setExpr(metricTypeParamsYamlTpl.getExpr());
metric.setMetricDefinition(MetricDefineType.METRIC, params);
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
MetricDefineByFieldParams params = new MetricDefineByFieldParams();
params.setExpr(metricTypeParamsYamlTpl.getExpr());
params.setFields(getFieldParams(metricTypeParamsYamlTpl.getFields()));
metric.setMetricDefinition(MetricDefineType.FIELD, params);
}
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
metricTypeParams.setFieldMetric(true);
}
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
metricTypeParams.setFieldMetric(true);
}
return metricTypeParams;
}
private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
List<Measure> measures = new ArrayList<>();
private static List<FieldParam> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
List<FieldParam> fields = new ArrayList<>();
for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
Measure measure = new Measure();
measure.setName(fieldParamYamlTpl.getFieldName());
measure.setExpr(fieldParamYamlTpl.getFieldName());
measures.add(measure);
FieldParam field = new FieldParam();
field.setFieldName(fieldParamYamlTpl.getFieldName());
fields.add(field);
}
return measures;
return fields;
}
private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
List<Measure> measures = new ArrayList<>();
private static List<MetricParam> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
List<MetricParam> metrics = new ArrayList<>();
for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
Measure measure = new Measure();
measure.setName(metricParamYamlTpl.getBizName());
measure.setExpr(metricParamYamlTpl.getBizName());
measures.add(measure);
MetricParam metric = new MetricParam();
metric.setBizName(metricParamYamlTpl.getBizName());
metric.setId(metricParamYamlTpl.getId());
metrics.add(metric);
}
return measures;
return metrics;
}
private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
List<Measure> measures = new ArrayList<>();
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
Measure measure = new Measure();
measure.setCreateMetric(measureYamlTpl.getCreateMetric());
measure.setExpr(measureYamlTpl.getExpr());
measure.setAgg(measureYamlTpl.getAgg());
measure.setName(measureYamlTpl.getName());
@@ -170,28 +174,30 @@ public class SemanticSchemaManager {
private static List<Dimension> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
List<Dimension> dimensions = new ArrayList<>();
for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) {
Dimension dimension = Dimension.builder().build();
dimension.setType(dimensionYamlTpl.getType());
Dimension dimension = new Dimension();
if (Objects.nonNull(dimensionYamlTpl.getType())) {
dimension.setType(DimensionType.valueOf(dimensionYamlTpl.getType()));
}
dimension.setExpr(dimensionYamlTpl.getExpr());
dimension.setName(dimensionYamlTpl.getName());
dimension.setOwners(dimensionYamlTpl.getOwners());
dimension.setBizName(dimensionYamlTpl.getBizName());
dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
}
if (Objects.isNull(dimension.getDataType())) {
dimension.setDataType(DataType.UNKNOWN);
}
if (Objects.nonNull(dimensionYamlTpl.getExt())) {
dimension.setExt(dimensionYamlTpl.getExt());
}
dimension.setDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams());
dimension.setTypeParams(dimensionYamlTpl.getTypeParams());
dimensions.add(dimension);
}
return dimensions;
}
private static DimensionTimeTypeParams getDimensionTimeTypeParams(
DimensionTimeTypeParams dimensionTimeTypeParamsTpl) {
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
if (dimensionTimeTypeParamsTpl != null) {
dimensionTimeTypeParams
.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
}
return dimensionTimeTypeParams;
}
private static List<Identify> getIdentify(List<IdentifyYamlTpl> identifyYamlTpls) {
List<Identify> identifies = new ArrayList<>();
for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) {
@@ -227,17 +233,18 @@ public class SemanticSchemaManager {
return joinRelations;
}
public static void update(S2CalciteSchema schema, List<Metric> metric) throws Exception {
public static void update(S2CalciteSchema schema, List<MetricSchemaResp> metric)
throws Exception {
if (schema != null) {
updateMetric(metric, schema.getMetrics());
}
}
public static void update(S2CalciteSchema schema, DataModel datasourceYamlTpl)
public static void update(S2CalciteSchema schema, ModelResp datasourceYamlTpl)
throws Exception {
if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataModel>> datasourceYamlTplMap =
Optional<Entry<String, ModelResp>> datasourceYamlTplMap =
schema.getDataModels().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
@@ -249,31 +256,31 @@ public class SemanticSchemaManager {
}
public static void update(S2CalciteSchema schema, String datasourceBizName,
List<Dimension> dimensionYamlTpls) throws Exception {
List<DimSchemaResp> dimensionYamlTpls) throws Exception {
if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
Optional<Map.Entry<String, List<DimSchemaResp>>> datasourceYamlTplMap = schema
.getDimensions().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
} else {
List<Dimension> dimensions = new ArrayList<>();
List<DimSchemaResp> dimensions = new ArrayList<>();
updateDimension(dimensionYamlTpls, dimensions);
schema.getDimensions().put(datasourceBizName, dimensions);
}
}
}
private static void updateDimension(List<Dimension> dimensionYamlTpls,
List<Dimension> dimensions) {
private static void updateDimension(List<DimSchemaResp> dimensionYamlTpls,
List<DimSchemaResp> dimensions) {
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
return;
}
Set<String> toAdd =
dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Iterator<Dimension> iterator = dimensions.iterator();
Iterator<DimSchemaResp> iterator = dimensions.iterator();
while (iterator.hasNext()) {
Dimension cur = iterator.next();
DimSchemaResp cur = iterator.next();
if (toAdd.contains(cur.getName())) {
iterator.remove();
}
@@ -281,15 +288,16 @@ public class SemanticSchemaManager {
dimensions.addAll(dimensionYamlTpls);
}
private static void updateMetric(List<Metric> metricYamlTpls, List<Metric> metrics) {
private static void updateMetric(List<MetricSchemaResp> metricYamlTpls,
List<MetricSchemaResp> metrics) {
if (CollectionUtils.isEmpty(metricYamlTpls)) {
return;
}
Set<String> toAdd =
metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Iterator<Metric> iterator = metrics.iterator();
Iterator<MetricSchemaResp> iterator = metrics.iterator();
while (iterator.hasNext()) {
Metric cur = iterator.next();
MetricSchemaResp cur = iterator.next();
if (toAdd.contains(cur.getName())) {
iterator.remove();
}

View File

@@ -15,6 +15,7 @@ public class QueryStatDO {
private String traceId;
private Long modelId;
private Long dataSetId;
@TableField("query_user")
private String queryUser;
private String createdAt;
/** corresponding type, such as sql, struct, etc */
@@ -27,7 +28,8 @@ public class QueryStatDO {
private String queryStructCmd;
@TableField("struct_cmd_md5")
private String queryStructCmdMd5;
private String querySql;
@TableField("query_sql")
private String sql;
private String sqlMd5;
private String queryEngine;
// private Long startTime;

View File

@@ -190,7 +190,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
private SemanticQueryResp queryWithColumns(String sql, DatabaseResp database) {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtils = this.sqlUtils.init(database);
log.info("query SQL: {}", sql);
log.info("query SQL: {}", StringUtils.normalizeSpace(sql));
sqlUtils.queryInternal(sql, queryResultWithColumns);
return queryResultWithColumns;
}

View File

@@ -62,22 +62,19 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
private DataSetService dataSetService;
private TagMetaService tagMetaService;
@Autowired
private ApplicationEventPublisher eventPublisher;
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
ModelRelaService modelRelaService, DataSetService dataSetService,
TagMetaService tagMetaService) {
ModelRelaService modelRelaService, DataSetService dataSetService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.aliasGenerateHelper = aliasGenerateHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.dataSetService = dataSetService;
this.tagMetaService = tagMetaService;
}
@Override

View File

@@ -240,7 +240,7 @@ public class DownloadServiceImpl implements DownloadService {
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
.collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
queryStructReq.getGroups().add(0, dateConf.getDateField());
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
@@ -250,10 +250,6 @@ public class DownloadServiceImpl implements DownloadService {
return queryStructReq;
}
private String getTimeDimension(DateConf dateConf) {
return dateConf.getDateField();
}
private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {
for (MetricResp metricResp : metricResps) {
List<DrillDownDimension> drillDownDimensions =

View File

@@ -8,7 +8,6 @@ import com.github.pagehelper.PageInfo;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.pojo.enums.*;
import com.tencent.supersonic.common.util.BeanMapper;
@@ -59,15 +58,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private ApplicationEventPublisher eventPublisher;
private TagMetaService tagMetaService;
private ChatLayerService chatLayerService;
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
DimensionService dimensionService, TagMetaService tagMetaService,
@Lazy ChatLayerService chatLayerService) {
DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.aliasGenerateHelper = aliasGenerateHelper;
@@ -75,7 +71,6 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
this.collectService = collectService;
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
this.tagMetaService = tagMetaService;
this.chatLayerService = chatLayerService;
}
@@ -386,7 +381,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<Measure> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
List<String> fieldNameDepended = measures.stream().map(Measure::getBizName)
List<String> fieldNameDepended = measures.stream().map(Measure::getName)
// measure bizName = model bizName_fieldName
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
.collect(Collectors.toList());
@@ -660,37 +655,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
&& !metricResp.getDefaultAgg().isEmpty())) {
return metricResp.getDefaultAgg();
}
// FIELD define will get from expr
if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
return SqlSelectFunctionHelper.getFirstAggregateFunctions(metricResp.getExpr());
}
// METRIC define will get from first metric
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
if (!CollectionUtils.isEmpty(metricResp.getMetricDefineByMetricParams().getMetrics())) {
MetricParam metricParam =
metricResp.getMetricDefineByMetricParams().getMetrics().get(0);
MetricResp firstMetricResp =
getMetric(modelResp.getDomainId(), metricParam.getBizName());
if (Objects.nonNull(firstMetricResp)) {
return getDefaultAgg(firstMetricResp, modelResp);
}
return "";
}
}
// Measure define will get from first measure
List<Measure> measures = modelResp.getModelDetail().getMeasures();
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
if (CollectionUtils.isEmpty(measureParams)) {
return "";
}
Measure firstMeasure = measureParams.get(0);
if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<Measure> measures = modelResp.getModelDetail().getMeasures();
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
if (CollectionUtils.isEmpty(measureParams)) {
return null;
}
Measure firstMeasure = measureParams.get(0);
for (Measure measure : measures) {
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
return measure.getAgg();
for (Measure measure : measures) {
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
return measure.getAgg();
}
}
}
return "";
return null;
}
@Override
@@ -716,31 +697,31 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryMetricReq.setDateInfo(null);
}
// 4. set groups
List<String> dimensionBizNames = dimensionResps.stream()
List<String> dimensionNames = dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
|| queryMetricReq.getDimensionNames().contains(entry.getBizName())
|| queryMetricReq.getDimensionIds().contains(entry.getId()))
.map(SchemaItem::getBizName).collect(Collectors.toList());
.map(SchemaItem::getName).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateInfo = queryMetricReq.getDateInfo();
if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) {
queryStructReq.getGroups().add(dateInfo.getDateField());
}
if (!CollectionUtils.isEmpty(dimensionBizNames)) {
queryStructReq.getGroups().addAll(dimensionBizNames);
if (!CollectionUtils.isEmpty(dimensionNames)) {
queryStructReq.getGroups().addAll(dimensionNames);
}
// 5. set aggregators
List<String> metricBizNames = metricResps.stream()
List<String> metricNames = metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(SchemaItem::getBizName).collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricBizNames)) {
.map(SchemaItem::getName).collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricNames)) {
throw new IllegalArgumentException(
"Invalid input parameters, unable to obtain valid metrics");
}
List<Aggregator> aggregators = new ArrayList<>();
for (String metricBizName : metricBizNames) {
for (String metricBizName : metricNames) {
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricBizName);
aggregators.add(aggregator);

View File

@@ -2,12 +2,15 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
@@ -51,7 +54,7 @@ import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -69,7 +72,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@@ -94,13 +100,16 @@ public class ModelServiceImpl implements ModelService {
private final ModelRelaService modelRelaService;
private final ThreadPoolExecutor executor;
private final ApplicationEventPublisher eventPublisher;
ExecutorService executor =
new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
DomainService domainService, UserService userService, DataSetService dataSetService,
DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService,
@Qualifier("commonExecutor") ThreadPoolExecutor executor) {
ApplicationEventPublisher eventPublisher) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
this.dimensionService = dimensionService;
@@ -110,17 +119,18 @@ public class ModelServiceImpl implements ModelService {
this.dataSetService = dataSetService;
this.dateInfoRepository = dateInfoRepository;
this.modelRelaService = modelRelaService;
this.executor = executor;
this.eventPublisher = eventPublisher;
}
@Override
@Transactional
public ModelResp createModel(ModelReq modelReq, User user) throws Exception {
// checkParams(modelReq);
checkParams(modelReq);
ModelDO modelDO = ModelConverter.convert(modelReq, user);
modelRepository.createModel(modelDO);
batchCreateDimension(modelDO, user);
batchCreateMetric(modelDO, user);
sendEvent(modelDO, EventType.ADD);
return ModelConverter.convert(modelDO);
}
@@ -140,13 +150,14 @@ public class ModelServiceImpl implements ModelService {
@Override
@Transactional
public ModelResp updateModel(ModelReq modelReq, User user) throws Exception {
// checkParams(modelReq);
checkParams(modelReq);
checkRelations(modelReq);
ModelDO modelDO = modelRepository.getModelById(modelReq.getId());
ModelConverter.convert(modelDO, modelReq, user);
modelRepository.updateModel(modelDO);
batchCreateDimension(modelDO, user);
batchCreateMetric(modelDO, user);
sendEvent(modelDO, EventType.UPDATE);
return ModelConverter.convert(modelDO);
}
@@ -607,4 +618,16 @@ public class ModelServiceImpl implements ModelService {
}
return false;
}
private void sendEvent(ModelDO modelDO, EventType eventType) {
DataItem dataItem = getDataItem(modelDO);
eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
}
private DataItem getDataItem(ModelDO modelDO) {
return DataItem.builder().id(modelDO.getId().toString()).name(modelDO.getName())
.bizName(modelDO.getBizName()).modelId(modelDO.getId().toString())
.domainId(modelDO.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
}
}

View File

@@ -2,17 +2,7 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DimensionConstants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -21,11 +11,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
public class DataSetSchemaBuilder {

View File

@@ -29,10 +29,11 @@ public class ModelConverter {
public static ModelDO convert(ModelReq modelReq, User user) {
ModelDO modelDO = new ModelDO();
// ModelDetail modelDetail = createModelDetail(modelReq);
ModelDetail modelDetail = modelReq.getModelDetail();
modelReq.createdBy(user.getName());
BeanMapper.mapper(modelReq, modelDO);
modelDO.setStatus(StatusEnum.ONLINE.getCode());
modelDO.setModelDetail(JSONObject.toJSONString(modelReq.getModelDetail()));
modelDO.setModelDetail(JSONObject.toJSONString(modelDetail));
modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions()));
if (modelReq.getExt() != null) {
modelDO.setExt(JSONObject.toJSONString(modelReq.getExt()));
@@ -123,9 +124,7 @@ public class ModelConverter {
metricReq.setModelId(modelDO.getId());
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
exprTypeParams.setExpr(measure.getExpr());
Measure measureParam = new Measure();
BeanMapper.mapper(measure, measureParam);
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
exprTypeParams.setMeasures(Lists.newArrayList(measure));
metricReq.setMetricDefineByMeasureParams(exprTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
return metricReq;
@@ -165,11 +164,14 @@ public class ModelConverter {
getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
modelDetail.getIdentifiers().add(identify);
} else if (FieldType.measure.equals(fieldType)) {
Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
columnSchema.getAgg().getOperator(), 1);
Measure measure = new Measure(columnSchema.getName(),
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
modelDetail.getMeasures().add(measure);
} else {
Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
Dimension dim = new Dimension(columnSchema.getName(),
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
columnSchema.getColumnName(),
DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
modelDetail.getDimensions().add(dim);
}
@@ -264,14 +266,17 @@ public class ModelConverter {
private static ModelDetail createModelDetail(ModelReq modelReq) {
ModelDetail modelDetail = new ModelDetail();
// List<Measure> measures = modelReq.getModelDetail().getMeasures();
// for (Measure measure : measures) {
// if (StringUtils.isBlank(measure.getBizName())) {
// continue;
// }
// measure.setExpr(measure.getBizName());
// measure.setBizName(String.format("%s_%", modelReq.getBizName(), measure.getExpr()));
// }
List<Measure> measures = modelReq.getModelDetail().getMeasures();
if (measures == null) {
measures = Lists.newArrayList();
}
for (Measure measure : measures) {
if (StringUtils.isBlank(measure.getBizName())) {
continue;
}
measure.setExpr(measure.getBizName());
measure.setBizName(String.format("%s_%s", modelReq.getBizName(), measure.getExpr()));
}
BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
return modelDetail;
}

View File

@@ -1,244 +1,230 @@
package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@Slf4j
class HeadlessParserServiceTest {
public static SqlParserResp parser(S2CalciteSchema semanticSchema, OntologyQuery ontologyQuery,
boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp();
try {
if (semanticSchema == null) {
sqlParser.setErrMsg("headlessSchema not found");
return sqlParser;
}
SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setOntologyQuery(ontologyQuery);
String sql = aggBuilder.buildOntologySql(queryStatement);
queryStatement.setSql(sql);
EngineType engineType = semanticSchema.getOntology().getDatabaseType();
sqlParser.setSql(aggBuilder.getSql(engineType));
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", ontologyQuery, e);
}
return sqlParser;
}
public void test() throws Exception {
DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("s2_pv_uv_statis");
datasource.setSourceId(1L);
datasource.setSqlQuery(
"SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
MeasureYamlTpl measure = new MeasureYamlTpl();
measure.setAgg("sum");
measure.setName("s2_pv_uv_statis_pv");
measure.setExpr("pv");
List<MeasureYamlTpl> measures = new ArrayList<>();
measures.add(measure);
MeasureYamlTpl measure2 = new MeasureYamlTpl();
measure2.setAgg("count");
measure2.setName("s2_pv_uv_statis_internal_cnt");
measure2.setExpr("1");
measure2.setCreateMetric("true");
measures.add(measure2);
MeasureYamlTpl measure3 = new MeasureYamlTpl();
measure3.setAgg("count");
measure3.setName("s2_pv_uv_statis_uv");
measure3.setExpr("uv");
measure3.setCreateMetric("true");
measures.add(measure3);
datasource.setMeasures(measures);
DimensionYamlTpl dimension = new DimensionYamlTpl();
dimension.setName("imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
List<DimensionYamlTpl> dimensions = new ArrayList<>();
dimensions.add(dimension);
DimensionYamlTpl dimension2 = new DimensionYamlTpl();
dimension2.setName("sys_imp_date");
dimension2.setExpr("imp_date");
dimension2.setType("time");
DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams();
dimensionTimeTypeParams2.setIsPrimary("true");
dimensionTimeTypeParams2.setTimeGranularity("day");
dimension2.setTypeParams(dimensionTimeTypeParams2);
dimensions.add(dimension2);
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("day");
dimension3.setTypeParams(dimensionTimeTypeParams3);
dimensions.add(dimension3);
datasource.setDimensions(dimensions);
List<IdentifyYamlTpl> identifies = new ArrayList<>();
IdentifyYamlTpl identify = new IdentifyYamlTpl();
identify.setName("user_name");
identify.setType("primary");
identifies.add(identify);
datasource.setIdentifiers(identifies);
S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
SemanticSchemaManager.update(semanticSchema,
SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page");
dimension1.setName("page");
dimension1.setType("categorical");
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
MetricYamlTpl metric1 = new MetricYamlTpl();
metric1.setName("pv");
metric1.setType("expr");
MetricTypeParamsYamlTpl metricTypeParams = new MetricTypeParamsYamlTpl();
List<MeasureYamlTpl> measures1 = new ArrayList<>();
MeasureYamlTpl measure1 = new MeasureYamlTpl();
measure1.setName("s2_pv_uv_statis_pv");
measures1.add(measure1);
metricTypeParams.setMeasures(measures1);
metricTypeParams.setExpr("s2_pv_uv_statis_pv");
metric1.setTypeParams(metricTypeParams);
List<MetricYamlTpl> metric = new ArrayList<>();
metric.add(metric1);
MetricYamlTpl metric2 = new MetricYamlTpl();
metric2.setName("uv");
metric2.setType("expr");
MetricTypeParamsYamlTpl metricTypeParams1 = new MetricTypeParamsYamlTpl();
List<MeasureYamlTpl> measures2 = new ArrayList<>();
MeasureYamlTpl measure4 = new MeasureYamlTpl();
measure4.setName("s2_pv_uv_statis_uv");
measures2.add(measure4);
metricTypeParams1.setMeasures(measures2);
metricTypeParams1.setExpr("s2_pv_uv_statis_uv");
metric2.setTypeParams(metricTypeParams1);
metric.add(metric2);
// HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
OntologyQuery metricCommand = new OntologyQuery();
metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
metricCommand.setWhere(
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
metricCommand.setLimit(1000L);
List<ColumnOrder> orders = new ArrayList<>();
orders.add(ColumnOrder.buildDesc("sys_imp_date"));
metricCommand.setOrder(orders);
System.out.println(parser(semanticSchema, metricCommand, true));
addDepartment(semanticSchema);
OntologyQuery metricCommand2 = new OntologyQuery();
metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
"user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
metricCommand2.setWhere(
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
metricCommand2.setLimit(1000L);
List<ColumnOrder> orders2 = new ArrayList<>();
orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
metricCommand2.setOrder(orders2);
System.out.println(parser(semanticSchema, metricCommand2, true));
}
private static void addDepartment(S2CalciteSchema semanticSchema) {
DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("user_department");
datasource.setSourceId(1L);
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
MeasureYamlTpl measure = new MeasureYamlTpl();
measure.setAgg("count");
measure.setName("user_department_internal_cnt");
measure.setCreateMetric("true");
measure.setExpr("1");
List<MeasureYamlTpl> measures = new ArrayList<>();
measures.add(measure);
datasource.setMeasures(measures);
DimensionYamlTpl dimension = new DimensionYamlTpl();
dimension.setName("sys_imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
List<DimensionYamlTpl> dimensions = new ArrayList<>();
dimensions.add(dimension);
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("week");
dimension3.setTypeParams(dimensionTimeTypeParams3);
dimensions.add(dimension3);
datasource.setDimensions(dimensions);
List<IdentifyYamlTpl> identifies = new ArrayList<>();
IdentifyYamlTpl identify = new IdentifyYamlTpl();
identify.setName("user_name");
identify.setType("primary");
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDataModels().put("user_department",
SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
dimension1.setName("department");
dimension1.setType("categorical");
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimensions().put("user_department",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
}
//
// public static SqlParserResp parser(S2CalciteSchema semanticSchema, OntologyQuery
// ontologyQuery,
// boolean isAgg) {
// SqlParserResp sqlParser = new SqlParserResp();
// try {
// if (semanticSchema == null) {
// sqlParser.setErrMsg("headlessSchema not found");
// return sqlParser;
// }
// SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
// QueryStatement queryStatement = new QueryStatement();
// queryStatement.setOntologyQuery(ontologyQuery);
// String sql = aggBuilder.buildOntologySql(queryStatement);
// queryStatement.setSql(sql);
// EngineType engineType = semanticSchema.getOntology().getDatabase().getType();
// sqlParser.setSql(aggBuilder.getSql(engineType));
// } catch (Exception e) {
// sqlParser.setErrMsg(e.getMessage());
// log.error("parser error metricQueryReq[{}] error [{}]", ontologyQuery, e);
// }
// return sqlParser;
// }
//
// public void test() throws Exception {
//
// DataModelYamlTpl datasource = new DataModelYamlTpl();
// datasource.setName("s2_pv_uv_statis");
// datasource.setSourceId(1L);
// datasource.setSqlQuery(
// "SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
//
// MeasureYamlTpl measure = new MeasureYamlTpl();
// measure.setAgg("sum");
// measure.setName("s2_pv_uv_statis_pv");
// measure.setExpr("pv");
// List<MeasureYamlTpl> measures = new ArrayList<>();
// measures.add(measure);
//
// MeasureYamlTpl measure2 = new MeasureYamlTpl();
// measure2.setAgg("count");
// measure2.setName("s2_pv_uv_statis_internal_cnt");
// measure2.setExpr("1");
// measure2.setCreateMetric("true");
// measures.add(measure2);
//
// MeasureYamlTpl measure3 = new MeasureYamlTpl();
// measure3.setAgg("count");
// measure3.setName("s2_pv_uv_statis_uv");
// measure3.setExpr("uv");
// measure3.setCreateMetric("true");
// measures.add(measure3);
//
// datasource.setMeasures(measures);
//
// DimensionYamlTpl dimension = new DimensionYamlTpl();
// dimension.setName("imp_date");
// dimension.setExpr("imp_date");
// dimension.setType("time");
// DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
// dimensionTimeTypeParams.setIsPrimary("true");
// dimensionTimeTypeParams.setTimeGranularity("day");
// dimension.setTypeParams(dimensionTimeTypeParams);
// List<DimensionYamlTpl> dimensions = new ArrayList<>();
// dimensions.add(dimension);
//
// DimensionYamlTpl dimension2 = new DimensionYamlTpl();
// dimension2.setName("sys_imp_date");
// dimension2.setExpr("imp_date");
// dimension2.setType("time");
// DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams();
// dimensionTimeTypeParams2.setIsPrimary("true");
// dimensionTimeTypeParams2.setTimeGranularity("day");
// dimension2.setTypeParams(dimensionTimeTypeParams2);
// dimensions.add(dimension2);
//
// DimensionYamlTpl dimension3 = new DimensionYamlTpl();
// dimension3.setName("sys_imp_week");
// dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
// dimension3.setType("time");
// DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
// dimensionTimeTypeParams3.setIsPrimary("true");
// dimensionTimeTypeParams3.setTimeGranularity("day");
// dimension3.setTypeParams(dimensionTimeTypeParams3);
// dimensions.add(dimension3);
//
// datasource.setDimensions(dimensions);
//
// List<IdentifyYamlTpl> identifies = new ArrayList<>();
// IdentifyYamlTpl identify = new IdentifyYamlTpl();
// identify.setName("user_name");
// identify.setType("primary");
// identifies.add(identify);
// datasource.setIdentifiers(identifies);
// S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
//
// SemanticSchemaManager.update(semanticSchema,
// SemanticSchemaManager.getDataModel(datasource));
//
// DimensionYamlTpl dimension1 = new DimensionYamlTpl();
// dimension1.setExpr("page");
// dimension1.setName("page");
// dimension1.setType("categorical");
// List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
// dimensionYamlTpls.add(dimension1);
//
// SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
// SemanticSchemaManager.getDimensions(dimensionYamlTpls));
//
// MetricYamlTpl metric1 = new MetricYamlTpl();
// metric1.setName("pv");
// metric1.setType("expr");
// MetricTypeParamsYamlTpl metricTypeParams = new MetricTypeParamsYamlTpl();
// List<MeasureYamlTpl> measures1 = new ArrayList<>();
// MeasureYamlTpl measure1 = new MeasureYamlTpl();
// measure1.setName("s2_pv_uv_statis_pv");
// measures1.add(measure1);
// metricTypeParams.setMeasures(measures1);
// metricTypeParams.setExpr("s2_pv_uv_statis_pv");
// metric1.setTypeParams(metricTypeParams);
// List<MetricYamlTpl> metric = new ArrayList<>();
// metric.add(metric1);
//
// MetricYamlTpl metric2 = new MetricYamlTpl();
// metric2.setName("uv");
// metric2.setType("expr");
// MetricTypeParamsYamlTpl metricTypeParams1 = new MetricTypeParamsYamlTpl();
// List<MeasureYamlTpl> measures2 = new ArrayList<>();
// MeasureYamlTpl measure4 = new MeasureYamlTpl();
// measure4.setName("s2_pv_uv_statis_uv");
// measures2.add(measure4);
// metricTypeParams1.setMeasures(measures2);
// metricTypeParams1.setExpr("s2_pv_uv_statis_uv");
// metric2.setTypeParams(metricTypeParams1);
// metric.add(metric2);
//
// // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
//
// OntologyQuery metricCommand = new OntologyQuery();
// metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
// metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
// metricCommand.setWhere(
// "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
// metricCommand.setLimit(1000L);
// List<ColumnOrder> orders = new ArrayList<>();
// orders.add(ColumnOrder.buildDesc("sys_imp_date"));
// metricCommand.setOrder(orders);
// System.out.println(parser(semanticSchema, metricCommand, true));
//
// addDepartment(semanticSchema);
//
// OntologyQuery metricCommand2 = new OntologyQuery();
// metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
// "user_name__department", "user_name", "user_name__page")));
// metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
// metricCommand2.setWhere(
// "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
// metricCommand2.setLimit(1000L);
// List<ColumnOrder> orders2 = new ArrayList<>();
// orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
// metricCommand2.setOrder(orders2);
// System.out.println(parser(semanticSchema, metricCommand2, true));
// }
//
// private static void addDepartment(S2CalciteSchema semanticSchema) {
// DataModelYamlTpl datasource = new DataModelYamlTpl();
// datasource.setName("user_department");
// datasource.setSourceId(1L);
// datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
//
// MeasureYamlTpl measure = new MeasureYamlTpl();
// measure.setAgg("count");
// measure.setName("user_department_internal_cnt");
// measure.setCreateMetric("true");
// measure.setExpr("1");
// List<MeasureYamlTpl> measures = new ArrayList<>();
// measures.add(measure);
//
// datasource.setMeasures(measures);
//
// DimensionYamlTpl dimension = new DimensionYamlTpl();
// dimension.setName("sys_imp_date");
// dimension.setExpr("imp_date");
// dimension.setType("time");
// DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
// dimensionTimeTypeParams.setIsPrimary("true");
// dimensionTimeTypeParams.setTimeGranularity("day");
// dimension.setTypeParams(dimensionTimeTypeParams);
// List<DimensionYamlTpl> dimensions = new ArrayList<>();
// dimensions.add(dimension);
//
// DimensionYamlTpl dimension3 = new DimensionYamlTpl();
// dimension3.setName("sys_imp_week");
// dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
// dimension3.setType("time");
// DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
// dimensionTimeTypeParams3.setIsPrimary("true");
// dimensionTimeTypeParams3.setTimeGranularity("week");
// dimension3.setTypeParams(dimensionTimeTypeParams3);
// dimensions.add(dimension3);
//
// datasource.setDimensions(dimensions);
//
// List<IdentifyYamlTpl> identifies = new ArrayList<>();
// IdentifyYamlTpl identify = new IdentifyYamlTpl();
// identify.setName("user_name");
// identify.setType("primary");
// identifies.add(identify);
// datasource.setIdentifiers(identifies);
//
// semanticSchema.getDataModels().put("user_department",
// SemanticSchemaManager.getDataModel(datasource));
//
// DimensionYamlTpl dimension1 = new DimensionYamlTpl();
// dimension1.setExpr("department");
// dimension1.setName("department");
// dimension1.setType("categorical");
// List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
// dimensionYamlTpls.add(dimension1);
//
// semanticSchema.getDimensions().put("user_department",
// SemanticSchemaManager.getDimensions(dimensionYamlTpls));
// }
}

View File

@@ -0,0 +1,158 @@
package com.tencent.supersonic.headless.server.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.HashMap;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class MetricServiceImplTest {
//
// @Test
// void createMetric() throws Exception {
// MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
// ModelService modelService = Mockito.mock(ModelService.class);
// MetricService metricService = mockMetricService(metricRepository, modelService);
// MetricReq metricReq = buildMetricReq();
// when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
// when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
// MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getDefaultUser());
// MetricResp expectedMetricResp = buildExpectedMetricResp();
// Assertions.assertEquals(expectedMetricResp, actualMetricResp);
// }
//
// @Test
// void updateMetric() throws Exception {
// MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
// ModelService modelService = Mockito.mock(ModelService.class);
// MetricService metricService = mockMetricService(metricRepository, modelService);
// MetricReq metricReq = buildMetricUpdateReq();
// when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
// when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
// MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq());
// when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO);
// MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getDefaultUser());
// MetricResp expectedMetricResp = buildExpectedMetricResp();
// Assertions.assertEquals(expectedMetricResp, actualMetricResp);
// }
//
// private MetricService mockMetricService(MetricRepository metricRepository,
// ModelService modelService) {
// AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class);
// CollectService collectService = Mockito.mock(CollectService.class);
// ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
// DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
// DimensionService dimensionService = Mockito.mock(DimensionService.class);
// ChatLayerService chatLayerService = Mockito.mock(ChatLayerService.class);
// return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
// collectService, dataSetService, eventPublisher, dimensionService, chatLayerService);
// }
//
// private MetricReq buildMetricReq() {
// MetricReq metricReq = new MetricReq();
// metricReq.setId(1L);
// metricReq.setName("hr部门的访问次数");
// metricReq.setBizName("pv");
// metricReq.setDescription("SuperSonic的访问情况");
// metricReq.setAlias("pv");
// metricReq.setMetricDefineType(MetricDefineType.MEASURE);
// metricReq.setModelId(2L);
// metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
// DataFormat dataFormat = new DataFormat();
// dataFormat.setDecimalPlaces(3);
// dataFormat.setNeedMultiply100(false);
// metricReq.setDataFormat(dataFormat);
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
// new MeasureParam("s2_uv", "department='hr'")));
// typeParams.setExpr("s2_pv/s2_uv");
// metricReq.setMetricDefineByMeasureParams(typeParams);
// metricReq.setClassifications(Lists.newArrayList("核心指标"));
// metricReq.setRelateDimension(RelateDimension.builder().drillDownDimensions(
// Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
// .build());
// metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
// metricReq.setExt(new HashMap<>());
// return metricReq;
// }
//
// private MetricResp buildExpectedMetricResp() {
// MetricResp metricResp = new MetricResp();
// metricResp.setId(1L);
// metricResp.setName("hr部门的访问次数");
// metricResp.setBizName("pv");
// metricResp.setDescription("SuperSonic的访问情况");
// metricResp.setAlias("pv");
// metricResp.setMetricDefineType(MetricDefineType.MEASURE);
// metricResp.setModelId(2L);
// metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
// DataFormat dataFormat = new DataFormat();
// dataFormat.setDecimalPlaces(3);
// dataFormat.setNeedMultiply100(false);
// metricResp.setDataFormat(dataFormat);
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
// new MeasureParam("s2_uv", "department='hr'")));
// typeParams.setExpr("s2_pv/s2_uv");
// metricResp.setMetricDefineByMeasureParams(typeParams);
// metricResp.setClassifications("核心指标");
// metricResp.setRelateDimension(RelateDimension.builder().drillDownDimensions(
// Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
// .build());
// metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
// metricResp.setExt(new HashMap<>());
// metricResp.setTypeEnum(TypeEnums.METRIC);
// metricResp.setIsCollect(false);
// metricResp.setType(MetricType.DERIVED.name());
// metricResp.setStatus(StatusEnum.ONLINE.getCode());
// return metricResp;
// }
//
// private MetricReq buildMetricUpdateReq() {
// MetricReq metricReq = new MetricReq();
// metricReq.setId(1L);
// metricReq.setName("hr部门的访问次数");
// metricReq.setBizName("pv");
// metricReq.setMetricDefineType(MetricDefineType.MEASURE);
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
// new MeasureParam("s2_uv", "department='hr'")));
// typeParams.setExpr("s2_pv/s2_uv");
// metricReq.setMetricDefineByMeasureParams(typeParams);
// return metricReq;
// }
//
// private ModelResp mockModelResp() {
// ModelResp modelResp = new ModelResp();
// modelResp.setId(2L);
// modelResp.setDomainId(1L);
// return modelResp;
// }
}

View File

@@ -22,10 +22,10 @@ import com.tencent.supersonic.headless.server.utils.ModelConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
import static org.mockito.Mockito.when;
@@ -77,10 +77,10 @@ class ModelServiceImplTest {
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
DataSetService viewService = Mockito.mock(DataSetService.class);
ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class);
ThreadPoolExecutor threadPoolExecutor = Mockito.mock(ThreadPoolExecutor.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
return new ModelServiceImpl(modelRepository, databaseService, dimensionService,
metricService, domainService, userService, viewService, dateInfoRepository,
modelRelaService, threadPoolExecutor);
modelRelaService, eventPublisher);
}
private ModelReq mockModelReq() {