mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 12:07:42 +00:00
[improvement][headless]Release brand new version of Translator module.
This commit is contained in:
@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.*;
|
||||
@@ -30,6 +31,7 @@ import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||
import lombok.SneakyThrows;
|
||||
@@ -52,6 +54,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
private final DataSetService dataSetService;
|
||||
private final SchemaService schemaService;
|
||||
private final SemanticTranslator semanticTranslator;
|
||||
private final MetricDrillDownChecker metricDrillDownChecker;
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final MetricService metricService;
|
||||
private final DimensionService dimensionService;
|
||||
@@ -61,6 +64,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils,
|
||||
SemanticSchemaManager semanticSchemaManager, DataSetService dataSetService,
|
||||
SchemaService schemaService, SemanticTranslator semanticTranslator,
|
||||
MetricDrillDownChecker metricDrillDownChecker,
|
||||
KnowledgeBaseService knowledgeBaseService, MetricService metricService,
|
||||
DimensionService dimensionService) {
|
||||
this.statUtils = statUtils;
|
||||
@@ -69,6 +73,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
this.dataSetService = dataSetService;
|
||||
this.schemaService = schemaService;
|
||||
this.semanticTranslator = semanticTranslator;
|
||||
this.metricDrillDownChecker = metricDrillDownChecker;
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.metricService = metricService;
|
||||
this.dimensionService = dimensionService;
|
||||
@@ -115,6 +120,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
|
||||
semanticTranslator.translate(queryStatement);
|
||||
|
||||
// Check whether the dimensions of the metric drill-down are correct temporarily,
|
||||
// add the abstraction of a validator later.
|
||||
metricDrillDownChecker.checkQuery(queryStatement);
|
||||
|
||||
// 4.execute query
|
||||
SemanticQueryResp queryResp = null;
|
||||
for (QueryExecutor queryExecutor : queryExecutors) {
|
||||
@@ -198,6 +207,14 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
ModelResp modelResp = modelResps.get(0);
|
||||
String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
|
||||
modelResp.getName());
|
||||
List<Dimension> timeDims = modelResp.getTimeDimension();
|
||||
if (CollectionUtils.isNotEmpty(timeDims)) {
|
||||
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
|
||||
queryDimValueReq.getDateInfo().getDateField(),
|
||||
queryDimValueReq.getDateInfo().getStartDate(),
|
||||
queryDimValueReq.getDateInfo().getDateField(),
|
||||
queryDimValueReq.getDateInfo().getEndDate());
|
||||
}
|
||||
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
|
||||
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
|
||||
+ queryDimValueReq.getValue() + "%'";
|
||||
@@ -269,10 +286,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
|
||||
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
|
||||
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
|
||||
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
|
||||
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
|
||||
queryStatement.setIsTranslated(true);
|
||||
}
|
||||
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
|
||||
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
package com.tencent.supersonic.headless.server.manager;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
||||
import com.tencent.supersonic.headless.core.pojo.JoinRelation;
|
||||
import com.tencent.supersonic.headless.core.pojo.Ontology;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.s2sql.*;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.*;
|
||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -32,6 +34,24 @@ public class SemanticSchemaManager {
|
||||
|
||||
public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) {
|
||||
Ontology ontology = new Ontology();
|
||||
Map<String, List<MetricSchemaResp>> model2Metrics = Maps.newHashMap();
|
||||
semanticSchemaResp.getMetrics().forEach(dim -> {
|
||||
if (!model2Metrics.containsKey(dim.getModelBizName())) {
|
||||
model2Metrics.put(dim.getModelBizName(), Lists.newArrayList());
|
||||
}
|
||||
model2Metrics.get(dim.getModelBizName()).add(dim);
|
||||
});
|
||||
ontology.setMetricMap(model2Metrics);
|
||||
|
||||
Map<String, List<DimSchemaResp>> model2Dimensions = Maps.newHashMap();
|
||||
semanticSchemaResp.getDimensions().forEach(dim -> {
|
||||
if (!model2Dimensions.containsKey(dim.getModelBizName())) {
|
||||
model2Dimensions.put(dim.getModelBizName(), Lists.newArrayList());
|
||||
}
|
||||
model2Dimensions.get(dim.getModelBizName()).add(dim);
|
||||
});
|
||||
ontology.setDimensionMap(model2Dimensions);
|
||||
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
|
||||
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
|
||||
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
|
||||
@@ -45,25 +65,16 @@ public class SemanticSchemaManager {
|
||||
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
|
||||
}
|
||||
if (!dataModelYamlTpls.isEmpty()) {
|
||||
Map<String, DataModel> dataModelMap =
|
||||
Map<String, ModelResp> dataModelMap =
|
||||
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect(
|
||||
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
|
||||
ontology.setDataModelMap(dataModelMap);
|
||||
}
|
||||
if (!dimensionYamlTpls.isEmpty()) {
|
||||
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
|
||||
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
|
||||
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
|
||||
}
|
||||
ontology.setDimensionMap(dimensionMap);
|
||||
}
|
||||
if (!metricYamlTpls.isEmpty()) {
|
||||
ontology.setMetrics(getMetrics(metricYamlTpls));
|
||||
Collectors.toMap(ModelResp::getName, item -> item, (k1, k2) -> k1));
|
||||
ontology.setModelMap(dataModelMap);
|
||||
}
|
||||
|
||||
return ontology;
|
||||
}
|
||||
|
||||
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
||||
public static List<MetricSchemaResp> getMetrics(final List<MetricYamlTpl> t) {
|
||||
return getMetricsByMetricYamlTpl(t);
|
||||
}
|
||||
|
||||
@@ -71,36 +82,34 @@ public class SemanticSchemaManager {
|
||||
return getDimension(t);
|
||||
}
|
||||
|
||||
public static DataModel getDataModel(final DataModelYamlTpl d) {
|
||||
DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
|
||||
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
|
||||
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
|
||||
.measures(getMeasureParams(d.getMeasures()))
|
||||
.dimensions(getDimensions(d.getDimensions())).build();
|
||||
dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions()));
|
||||
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
|
||||
dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
|
||||
}
|
||||
public static ModelResp getDataModel(final DataModelYamlTpl d) {
|
||||
// ModelResp dataModel = ModelResp.builder()(d.getId()).modelId(d.getSourceId())
|
||||
// .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
|
||||
// .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
|
||||
// .measures(getMeasureParams(d.getMeasures()))
|
||||
// .dimensions(getDimensions(d.getDimensions())).build();
|
||||
ModelResp dataModel = new ModelResp();
|
||||
dataModel.setId(d.getId());
|
||||
dataModel.setName(d.getName());
|
||||
ModelDetail modelDetail = new ModelDetail();
|
||||
dataModel.setModelDetail(modelDetail);
|
||||
|
||||
modelDetail.setDbType(d.getType());
|
||||
modelDetail.setSqlQuery(d.getSqlQuery());
|
||||
modelDetail.setTableQuery(d.getTableQuery());
|
||||
modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
|
||||
modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
|
||||
modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
|
||||
|
||||
return dataModel;
|
||||
}
|
||||
|
||||
private static String getDataModelAggTime(List<Dimension> dimensions) {
|
||||
Optional<Dimension> timeDimension = dimensions.stream()
|
||||
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
|
||||
.findFirst();
|
||||
if (timeDimension.isPresent()
|
||||
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
|
||||
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
|
||||
}
|
||||
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
|
||||
}
|
||||
|
||||
private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
|
||||
List<Metric> metrics = new ArrayList<>();
|
||||
private static List<MetricSchemaResp> getMetricsByMetricYamlTpl(
|
||||
List<MetricYamlTpl> metricYamlTpls) {
|
||||
List<MetricSchemaResp> metrics = new ArrayList<>();
|
||||
for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {
|
||||
Metric metric = new Metric();
|
||||
metric.setMetricTypeParams(getMetricTypeParams(metricYamlTpl.getTypeParams()));
|
||||
metric.setOwners(metricYamlTpl.getOwners());
|
||||
MetricSchemaResp metric = new MetricSchemaResp();
|
||||
fillMetricTypeParams(metric, metricYamlTpl.getTypeParams());
|
||||
metric.setType(metricYamlTpl.getType());
|
||||
metric.setName(metricYamlTpl.getName());
|
||||
metrics.add(metric);
|
||||
@@ -108,55 +117,50 @@ public class SemanticSchemaManager {
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static MetricTypeParams getMetricTypeParams(
|
||||
private static void fillMetricTypeParams(MetricSchemaResp metric,
|
||||
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
|
||||
MetricTypeParams metricTypeParams = new MetricTypeParams();
|
||||
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
metricTypeParams.setFieldMetric(false);
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
|
||||
metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
|
||||
MetricDefineByMeasureParams params = new MetricDefineByMeasureParams();
|
||||
params.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
|
||||
metric.setMetricDefinition(MetricDefineType.MEASURE, params);
|
||||
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
|
||||
MetricDefineByMetricParams params = new MetricDefineByMetricParams();
|
||||
params.setMetrics(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
|
||||
params.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
metric.setMetricDefinition(MetricDefineType.METRIC, params);
|
||||
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
|
||||
MetricDefineByFieldParams params = new MetricDefineByFieldParams();
|
||||
params.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
params.setFields(getFieldParams(metricTypeParamsYamlTpl.getFields()));
|
||||
metric.setMetricDefinition(MetricDefineType.FIELD, params);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
|
||||
metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
|
||||
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
|
||||
metricTypeParams.setFieldMetric(true);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
|
||||
metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
|
||||
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
|
||||
metricTypeParams.setFieldMetric(true);
|
||||
}
|
||||
|
||||
return metricTypeParams;
|
||||
}
|
||||
|
||||
private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
private static List<FieldParam> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
|
||||
List<FieldParam> fields = new ArrayList<>();
|
||||
for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setName(fieldParamYamlTpl.getFieldName());
|
||||
measure.setExpr(fieldParamYamlTpl.getFieldName());
|
||||
measures.add(measure);
|
||||
FieldParam field = new FieldParam();
|
||||
field.setFieldName(fieldParamYamlTpl.getFieldName());
|
||||
fields.add(field);
|
||||
}
|
||||
return measures;
|
||||
return fields;
|
||||
}
|
||||
|
||||
private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
private static List<MetricParam> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
|
||||
List<MetricParam> metrics = new ArrayList<>();
|
||||
for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setName(metricParamYamlTpl.getBizName());
|
||||
measure.setExpr(metricParamYamlTpl.getBizName());
|
||||
measures.add(measure);
|
||||
MetricParam metric = new MetricParam();
|
||||
metric.setBizName(metricParamYamlTpl.getBizName());
|
||||
metric.setId(metricParamYamlTpl.getId());
|
||||
metrics.add(metric);
|
||||
}
|
||||
return measures;
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setCreateMetric(measureYamlTpl.getCreateMetric());
|
||||
measure.setExpr(measureYamlTpl.getExpr());
|
||||
measure.setAgg(measureYamlTpl.getAgg());
|
||||
measure.setName(measureYamlTpl.getName());
|
||||
@@ -170,28 +174,30 @@ public class SemanticSchemaManager {
|
||||
private static List<Dimension> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) {
|
||||
Dimension dimension = Dimension.builder().build();
|
||||
dimension.setType(dimensionYamlTpl.getType());
|
||||
Dimension dimension = new Dimension();
|
||||
if (Objects.nonNull(dimensionYamlTpl.getType())) {
|
||||
dimension.setType(DimensionType.valueOf(dimensionYamlTpl.getType()));
|
||||
}
|
||||
dimension.setExpr(dimensionYamlTpl.getExpr());
|
||||
dimension.setName(dimensionYamlTpl.getName());
|
||||
dimension.setOwners(dimensionYamlTpl.getOwners());
|
||||
dimension.setBizName(dimensionYamlTpl.getBizName());
|
||||
dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
|
||||
if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
|
||||
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
|
||||
}
|
||||
if (Objects.isNull(dimension.getDataType())) {
|
||||
dimension.setDataType(DataType.UNKNOWN);
|
||||
}
|
||||
if (Objects.nonNull(dimensionYamlTpl.getExt())) {
|
||||
dimension.setExt(dimensionYamlTpl.getExt());
|
||||
}
|
||||
dimension.setDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams());
|
||||
dimension.setTypeParams(dimensionYamlTpl.getTypeParams());
|
||||
dimensions.add(dimension);
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
private static DimensionTimeTypeParams getDimensionTimeTypeParams(
|
||||
DimensionTimeTypeParams dimensionTimeTypeParamsTpl) {
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
if (dimensionTimeTypeParamsTpl != null) {
|
||||
dimensionTimeTypeParams
|
||||
.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
|
||||
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
|
||||
}
|
||||
return dimensionTimeTypeParams;
|
||||
}
|
||||
|
||||
private static List<Identify> getIdentify(List<IdentifyYamlTpl> identifyYamlTpls) {
|
||||
List<Identify> identifies = new ArrayList<>();
|
||||
for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) {
|
||||
@@ -227,17 +233,18 @@ public class SemanticSchemaManager {
|
||||
return joinRelations;
|
||||
}
|
||||
|
||||
public static void update(S2CalciteSchema schema, List<Metric> metric) throws Exception {
|
||||
public static void update(S2CalciteSchema schema, List<MetricSchemaResp> metric)
|
||||
throws Exception {
|
||||
if (schema != null) {
|
||||
updateMetric(metric, schema.getMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(S2CalciteSchema schema, DataModel datasourceYamlTpl)
|
||||
public static void update(S2CalciteSchema schema, ModelResp datasourceYamlTpl)
|
||||
throws Exception {
|
||||
if (schema != null) {
|
||||
String dataSourceName = datasourceYamlTpl.getName();
|
||||
Optional<Entry<String, DataModel>> datasourceYamlTplMap =
|
||||
Optional<Entry<String, ModelResp>> datasourceYamlTplMap =
|
||||
schema.getDataModels().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
@@ -249,31 +256,31 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
|
||||
public static void update(S2CalciteSchema schema, String datasourceBizName,
|
||||
List<Dimension> dimensionYamlTpls) throws Exception {
|
||||
List<DimSchemaResp> dimensionYamlTpls) throws Exception {
|
||||
if (schema != null) {
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
|
||||
Optional<Map.Entry<String, List<DimSchemaResp>>> datasourceYamlTplMap = schema
|
||||
.getDimensions().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
|
||||
} else {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
List<DimSchemaResp> dimensions = new ArrayList<>();
|
||||
updateDimension(dimensionYamlTpls, dimensions);
|
||||
schema.getDimensions().put(datasourceBizName, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateDimension(List<Dimension> dimensionYamlTpls,
|
||||
List<Dimension> dimensions) {
|
||||
private static void updateDimension(List<DimSchemaResp> dimensionYamlTpls,
|
||||
List<DimSchemaResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd =
|
||||
dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Dimension> iterator = dimensions.iterator();
|
||||
Iterator<DimSchemaResp> iterator = dimensions.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Dimension cur = iterator.next();
|
||||
DimSchemaResp cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
@@ -281,15 +288,16 @@ public class SemanticSchemaManager {
|
||||
dimensions.addAll(dimensionYamlTpls);
|
||||
}
|
||||
|
||||
private static void updateMetric(List<Metric> metricYamlTpls, List<Metric> metrics) {
|
||||
private static void updateMetric(List<MetricSchemaResp> metricYamlTpls,
|
||||
List<MetricSchemaResp> metrics) {
|
||||
if (CollectionUtils.isEmpty(metricYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd =
|
||||
metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Metric> iterator = metrics.iterator();
|
||||
Iterator<MetricSchemaResp> iterator = metrics.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Metric cur = iterator.next();
|
||||
MetricSchemaResp cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ public class QueryStatDO {
|
||||
private String traceId;
|
||||
private Long modelId;
|
||||
private Long dataSetId;
|
||||
@TableField("query_user")
|
||||
private String queryUser;
|
||||
private String createdAt;
|
||||
/** corresponding type, such as sql, struct, etc */
|
||||
@@ -27,7 +28,8 @@ public class QueryStatDO {
|
||||
private String queryStructCmd;
|
||||
@TableField("struct_cmd_md5")
|
||||
private String queryStructCmdMd5;
|
||||
private String querySql;
|
||||
@TableField("query_sql")
|
||||
private String sql;
|
||||
private String sqlMd5;
|
||||
private String queryEngine;
|
||||
// private Long startTime;
|
||||
|
||||
@@ -190,7 +190,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
private SemanticQueryResp queryWithColumns(String sql, DatabaseResp database) {
|
||||
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
|
||||
SqlUtils sqlUtils = this.sqlUtils.init(database);
|
||||
log.info("query SQL: {}", sql);
|
||||
log.info("query SQL: {}", StringUtils.normalizeSpace(sql));
|
||||
sqlUtils.queryInternal(sql, queryResultWithColumns);
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
@@ -62,22 +62,19 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
|
||||
private DataSetService dataSetService;
|
||||
|
||||
private TagMetaService tagMetaService;
|
||||
|
||||
@Autowired
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
|
||||
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
|
||||
ModelRelaService modelRelaService, DataSetService dataSetService,
|
||||
TagMetaService tagMetaService) {
|
||||
ModelRelaService modelRelaService, DataSetService dataSetService) {
|
||||
this.modelService = modelService;
|
||||
this.dimensionRepository = dimensionRepository;
|
||||
this.aliasGenerateHelper = aliasGenerateHelper;
|
||||
this.databaseService = databaseService;
|
||||
this.modelRelaService = modelRelaService;
|
||||
this.dataSetService = dataSetService;
|
||||
this.tagMetaService = tagMetaService;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -240,7 +240,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
|
||||
.collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
queryStructReq.getGroups().add(0, dateConf.getDateField());
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
@@ -250,10 +250,6 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
return dateConf.getDateField();
|
||||
}
|
||||
|
||||
private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {
|
||||
for (MetricResp metricResp : metricResps) {
|
||||
List<DrillDownDimension> drillDownDimensions =
|
||||
|
||||
@@ -8,7 +8,6 @@ import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.pojo.*;
|
||||
import com.tencent.supersonic.common.pojo.enums.*;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
@@ -59,15 +58,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
|
||||
private TagMetaService tagMetaService;
|
||||
|
||||
private ChatLayerService chatLayerService;
|
||||
|
||||
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
|
||||
DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
|
||||
DimensionService dimensionService, TagMetaService tagMetaService,
|
||||
@Lazy ChatLayerService chatLayerService) {
|
||||
DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) {
|
||||
this.metricRepository = metricRepository;
|
||||
this.modelService = modelService;
|
||||
this.aliasGenerateHelper = aliasGenerateHelper;
|
||||
@@ -75,7 +71,6 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
this.collectService = collectService;
|
||||
this.dataSetService = dataSetService;
|
||||
this.dimensionService = dimensionService;
|
||||
this.tagMetaService = tagMetaService;
|
||||
this.chatLayerService = chatLayerService;
|
||||
}
|
||||
|
||||
@@ -386,7 +381,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
|
||||
List<Measure> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
List<String> fieldNameDepended = measures.stream().map(Measure::getBizName)
|
||||
List<String> fieldNameDepended = measures.stream().map(Measure::getName)
|
||||
// measure bizName = model bizName_fieldName
|
||||
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
|
||||
.collect(Collectors.toList());
|
||||
@@ -660,37 +655,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
&& !metricResp.getDefaultAgg().isEmpty())) {
|
||||
return metricResp.getDefaultAgg();
|
||||
}
|
||||
// FIELD define will get from expr
|
||||
if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
|
||||
return SqlSelectFunctionHelper.getFirstAggregateFunctions(metricResp.getExpr());
|
||||
}
|
||||
// METRIC define will get from first metric
|
||||
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
|
||||
if (!CollectionUtils.isEmpty(metricResp.getMetricDefineByMetricParams().getMetrics())) {
|
||||
MetricParam metricParam =
|
||||
metricResp.getMetricDefineByMetricParams().getMetrics().get(0);
|
||||
MetricResp firstMetricResp =
|
||||
getMetric(modelResp.getDomainId(), metricParam.getBizName());
|
||||
if (Objects.nonNull(firstMetricResp)) {
|
||||
return getDefaultAgg(firstMetricResp, modelResp);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
// Measure define will get from first measure
|
||||
List<Measure> measures = modelResp.getModelDetail().getMeasures();
|
||||
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
if (CollectionUtils.isEmpty(measureParams)) {
|
||||
return "";
|
||||
}
|
||||
Measure firstMeasure = measureParams.get(0);
|
||||
if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
|
||||
List<Measure> measures = modelResp.getModelDetail().getMeasures();
|
||||
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
if (CollectionUtils.isEmpty(measureParams)) {
|
||||
return null;
|
||||
}
|
||||
Measure firstMeasure = measureParams.get(0);
|
||||
|
||||
for (Measure measure : measures) {
|
||||
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
|
||||
return measure.getAgg();
|
||||
for (Measure measure : measures) {
|
||||
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
|
||||
return measure.getAgg();
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -716,31 +697,31 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
queryMetricReq.setDateInfo(null);
|
||||
}
|
||||
// 4. set groups
|
||||
List<String> dimensionBizNames = dimensionResps.stream()
|
||||
List<String> dimensionNames = dimensionResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
|
||||
|| queryMetricReq.getDimensionNames().contains(entry.getBizName())
|
||||
|| queryMetricReq.getDimensionIds().contains(entry.getId()))
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
.map(SchemaItem::getName).collect(Collectors.toList());
|
||||
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
DateConf dateInfo = queryMetricReq.getDateInfo();
|
||||
if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) {
|
||||
queryStructReq.getGroups().add(dateInfo.getDateField());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimensionBizNames)) {
|
||||
queryStructReq.getGroups().addAll(dimensionBizNames);
|
||||
if (!CollectionUtils.isEmpty(dimensionNames)) {
|
||||
queryStructReq.getGroups().addAll(dimensionNames);
|
||||
}
|
||||
// 5. set aggregators
|
||||
List<String> metricBizNames = metricResps.stream()
|
||||
List<String> metricNames = metricResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(metricBizNames)) {
|
||||
.map(SchemaItem::getName).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(metricNames)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Invalid input parameters, unable to obtain valid metrics");
|
||||
}
|
||||
List<Aggregator> aggregators = new ArrayList<>();
|
||||
for (String metricBizName : metricBizNames) {
|
||||
for (String metricBizName : metricNames) {
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricBizName);
|
||||
aggregators.add(aggregator);
|
||||
|
||||
@@ -2,12 +2,15 @@ package com.tencent.supersonic.headless.server.service.impl;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataItem;
|
||||
import com.tencent.supersonic.common.pojo.ItemDateResp;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.pojo.DBColumn;
|
||||
@@ -51,7 +54,7 @@ import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
@@ -69,7 +72,10 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@@ -94,13 +100,16 @@ public class ModelServiceImpl implements ModelService {
|
||||
|
||||
private final ModelRelaService modelRelaService;
|
||||
|
||||
private final ThreadPoolExecutor executor;
|
||||
private final ApplicationEventPublisher eventPublisher;
|
||||
|
||||
ExecutorService executor =
|
||||
new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
|
||||
|
||||
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
|
||||
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
|
||||
DomainService domainService, UserService userService, DataSetService dataSetService,
|
||||
DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService,
|
||||
@Qualifier("commonExecutor") ThreadPoolExecutor executor) {
|
||||
ApplicationEventPublisher eventPublisher) {
|
||||
this.modelRepository = modelRepository;
|
||||
this.databaseService = databaseService;
|
||||
this.dimensionService = dimensionService;
|
||||
@@ -110,17 +119,18 @@ public class ModelServiceImpl implements ModelService {
|
||||
this.dataSetService = dataSetService;
|
||||
this.dateInfoRepository = dateInfoRepository;
|
||||
this.modelRelaService = modelRelaService;
|
||||
this.executor = executor;
|
||||
this.eventPublisher = eventPublisher;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public ModelResp createModel(ModelReq modelReq, User user) throws Exception {
|
||||
// checkParams(modelReq);
|
||||
checkParams(modelReq);
|
||||
ModelDO modelDO = ModelConverter.convert(modelReq, user);
|
||||
modelRepository.createModel(modelDO);
|
||||
batchCreateDimension(modelDO, user);
|
||||
batchCreateMetric(modelDO, user);
|
||||
sendEvent(modelDO, EventType.ADD);
|
||||
return ModelConverter.convert(modelDO);
|
||||
}
|
||||
|
||||
@@ -140,13 +150,14 @@ public class ModelServiceImpl implements ModelService {
|
||||
@Override
|
||||
@Transactional
|
||||
public ModelResp updateModel(ModelReq modelReq, User user) throws Exception {
|
||||
// checkParams(modelReq);
|
||||
checkParams(modelReq);
|
||||
checkRelations(modelReq);
|
||||
ModelDO modelDO = modelRepository.getModelById(modelReq.getId());
|
||||
ModelConverter.convert(modelDO, modelReq, user);
|
||||
modelRepository.updateModel(modelDO);
|
||||
batchCreateDimension(modelDO, user);
|
||||
batchCreateMetric(modelDO, user);
|
||||
sendEvent(modelDO, EventType.UPDATE);
|
||||
return ModelConverter.convert(modelDO);
|
||||
}
|
||||
|
||||
@@ -607,4 +618,16 @@ public class ModelServiceImpl implements ModelService {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void sendEvent(ModelDO modelDO, EventType eventType) {
|
||||
DataItem dataItem = getDataItem(modelDO);
|
||||
eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
|
||||
}
|
||||
|
||||
private DataItem getDataItem(ModelDO modelDO) {
|
||||
return DataItem.builder().id(modelDO.getId().toString()).name(modelDO.getName())
|
||||
.bizName(modelDO.getBizName()).modelId(modelDO.getId().toString())
|
||||
.domainId(modelDO.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,17 +2,7 @@ package com.tencent.supersonic.headless.server.utils;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.DimensionConstants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
|
||||
@@ -21,11 +11,7 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class DataSetSchemaBuilder {
|
||||
|
||||
@@ -29,10 +29,11 @@ public class ModelConverter {
|
||||
public static ModelDO convert(ModelReq modelReq, User user) {
|
||||
ModelDO modelDO = new ModelDO();
|
||||
// ModelDetail modelDetail = createModelDetail(modelReq);
|
||||
ModelDetail modelDetail = modelReq.getModelDetail();
|
||||
modelReq.createdBy(user.getName());
|
||||
BeanMapper.mapper(modelReq, modelDO);
|
||||
modelDO.setStatus(StatusEnum.ONLINE.getCode());
|
||||
modelDO.setModelDetail(JSONObject.toJSONString(modelReq.getModelDetail()));
|
||||
modelDO.setModelDetail(JSONObject.toJSONString(modelDetail));
|
||||
modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions()));
|
||||
if (modelReq.getExt() != null) {
|
||||
modelDO.setExt(JSONObject.toJSONString(modelReq.getExt()));
|
||||
@@ -123,9 +124,7 @@ public class ModelConverter {
|
||||
metricReq.setModelId(modelDO.getId());
|
||||
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
|
||||
exprTypeParams.setExpr(measure.getExpr());
|
||||
Measure measureParam = new Measure();
|
||||
BeanMapper.mapper(measure, measureParam);
|
||||
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
|
||||
exprTypeParams.setMeasures(Lists.newArrayList(measure));
|
||||
metricReq.setMetricDefineByMeasureParams(exprTypeParams);
|
||||
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
|
||||
return metricReq;
|
||||
@@ -165,11 +164,14 @@ public class ModelConverter {
|
||||
getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
|
||||
modelDetail.getIdentifiers().add(identify);
|
||||
} else if (FieldType.measure.equals(fieldType)) {
|
||||
Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
|
||||
columnSchema.getAgg().getOperator(), 1);
|
||||
Measure measure = new Measure(columnSchema.getName(),
|
||||
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
|
||||
columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
|
||||
modelDetail.getMeasures().add(measure);
|
||||
} else {
|
||||
Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
|
||||
Dimension dim = new Dimension(columnSchema.getName(),
|
||||
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
|
||||
columnSchema.getColumnName(),
|
||||
DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
|
||||
modelDetail.getDimensions().add(dim);
|
||||
}
|
||||
@@ -264,14 +266,17 @@ public class ModelConverter {
|
||||
|
||||
private static ModelDetail createModelDetail(ModelReq modelReq) {
|
||||
ModelDetail modelDetail = new ModelDetail();
|
||||
// List<Measure> measures = modelReq.getModelDetail().getMeasures();
|
||||
// for (Measure measure : measures) {
|
||||
// if (StringUtils.isBlank(measure.getBizName())) {
|
||||
// continue;
|
||||
// }
|
||||
// measure.setExpr(measure.getBizName());
|
||||
// measure.setBizName(String.format("%s_%", modelReq.getBizName(), measure.getExpr()));
|
||||
// }
|
||||
List<Measure> measures = modelReq.getModelDetail().getMeasures();
|
||||
if (measures == null) {
|
||||
measures = Lists.newArrayList();
|
||||
}
|
||||
for (Measure measure : measures) {
|
||||
if (StringUtils.isBlank(measure.getBizName())) {
|
||||
continue;
|
||||
}
|
||||
measure.setExpr(measure.getBizName());
|
||||
measure.setBizName(String.format("%s_%s", modelReq.getBizName(), measure.getExpr()));
|
||||
}
|
||||
BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
|
||||
return modelDetail;
|
||||
}
|
||||
|
||||
@@ -1,244 +1,230 @@
|
||||
package com.tencent.supersonic.headless.server.calcite;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.common.pojo.enums.EngineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder;
|
||||
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
class HeadlessParserServiceTest {
|
||||
|
||||
public static SqlParserResp parser(S2CalciteSchema semanticSchema, OntologyQuery ontologyQuery,
|
||||
boolean isAgg) {
|
||||
SqlParserResp sqlParser = new SqlParserResp();
|
||||
try {
|
||||
if (semanticSchema == null) {
|
||||
sqlParser.setErrMsg("headlessSchema not found");
|
||||
return sqlParser;
|
||||
}
|
||||
SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setOntologyQuery(ontologyQuery);
|
||||
String sql = aggBuilder.buildOntologySql(queryStatement);
|
||||
queryStatement.setSql(sql);
|
||||
EngineType engineType = semanticSchema.getOntology().getDatabaseType();
|
||||
sqlParser.setSql(aggBuilder.getSql(engineType));
|
||||
} catch (Exception e) {
|
||||
sqlParser.setErrMsg(e.getMessage());
|
||||
log.error("parser error metricQueryReq[{}] error [{}]", ontologyQuery, e);
|
||||
}
|
||||
return sqlParser;
|
||||
}
|
||||
|
||||
public void test() throws Exception {
|
||||
|
||||
DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
datasource.setName("s2_pv_uv_statis");
|
||||
datasource.setSourceId(1L);
|
||||
datasource.setSqlQuery(
|
||||
"SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
|
||||
|
||||
MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
measure.setAgg("sum");
|
||||
measure.setName("s2_pv_uv_statis_pv");
|
||||
measure.setExpr("pv");
|
||||
List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
measures.add(measure);
|
||||
|
||||
MeasureYamlTpl measure2 = new MeasureYamlTpl();
|
||||
measure2.setAgg("count");
|
||||
measure2.setName("s2_pv_uv_statis_internal_cnt");
|
||||
measure2.setExpr("1");
|
||||
measure2.setCreateMetric("true");
|
||||
measures.add(measure2);
|
||||
|
||||
MeasureYamlTpl measure3 = new MeasureYamlTpl();
|
||||
measure3.setAgg("count");
|
||||
measure3.setName("s2_pv_uv_statis_uv");
|
||||
measure3.setExpr("uv");
|
||||
measure3.setCreateMetric("true");
|
||||
measures.add(measure3);
|
||||
|
||||
datasource.setMeasures(measures);
|
||||
|
||||
DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
dimension.setName("imp_date");
|
||||
dimension.setExpr("imp_date");
|
||||
dimension.setType("time");
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParams.setIsPrimary("true");
|
||||
dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
dimensions.add(dimension);
|
||||
|
||||
DimensionYamlTpl dimension2 = new DimensionYamlTpl();
|
||||
dimension2.setName("sys_imp_date");
|
||||
dimension2.setExpr("imp_date");
|
||||
dimension2.setType("time");
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParams2.setIsPrimary("true");
|
||||
dimensionTimeTypeParams2.setTimeGranularity("day");
|
||||
dimension2.setTypeParams(dimensionTimeTypeParams2);
|
||||
dimensions.add(dimension2);
|
||||
|
||||
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
dimension3.setName("sys_imp_week");
|
||||
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
dimension3.setType("time");
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
dimensionTimeTypeParams3.setTimeGranularity("day");
|
||||
dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
dimensions.add(dimension3);
|
||||
|
||||
datasource.setDimensions(dimensions);
|
||||
|
||||
List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
identify.setName("user_name");
|
||||
identify.setType("primary");
|
||||
identifies.add(identify);
|
||||
datasource.setIdentifiers(identifies);
|
||||
S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
|
||||
|
||||
SemanticSchemaManager.update(semanticSchema,
|
||||
SemanticSchemaManager.getDataModel(datasource));
|
||||
|
||||
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
dimension1.setExpr("page");
|
||||
dimension1.setName("page");
|
||||
dimension1.setType("categorical");
|
||||
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
dimensionYamlTpls.add(dimension1);
|
||||
|
||||
SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
|
||||
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
|
||||
MetricYamlTpl metric1 = new MetricYamlTpl();
|
||||
metric1.setName("pv");
|
||||
metric1.setType("expr");
|
||||
MetricTypeParamsYamlTpl metricTypeParams = new MetricTypeParamsYamlTpl();
|
||||
List<MeasureYamlTpl> measures1 = new ArrayList<>();
|
||||
MeasureYamlTpl measure1 = new MeasureYamlTpl();
|
||||
measure1.setName("s2_pv_uv_statis_pv");
|
||||
measures1.add(measure1);
|
||||
metricTypeParams.setMeasures(measures1);
|
||||
metricTypeParams.setExpr("s2_pv_uv_statis_pv");
|
||||
metric1.setTypeParams(metricTypeParams);
|
||||
List<MetricYamlTpl> metric = new ArrayList<>();
|
||||
metric.add(metric1);
|
||||
|
||||
MetricYamlTpl metric2 = new MetricYamlTpl();
|
||||
metric2.setName("uv");
|
||||
metric2.setType("expr");
|
||||
MetricTypeParamsYamlTpl metricTypeParams1 = new MetricTypeParamsYamlTpl();
|
||||
List<MeasureYamlTpl> measures2 = new ArrayList<>();
|
||||
MeasureYamlTpl measure4 = new MeasureYamlTpl();
|
||||
measure4.setName("s2_pv_uv_statis_uv");
|
||||
measures2.add(measure4);
|
||||
metricTypeParams1.setMeasures(measures2);
|
||||
metricTypeParams1.setExpr("s2_pv_uv_statis_uv");
|
||||
metric2.setTypeParams(metricTypeParams1);
|
||||
metric.add(metric2);
|
||||
|
||||
// HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
|
||||
|
||||
OntologyQuery metricCommand = new OntologyQuery();
|
||||
metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
|
||||
metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
|
||||
metricCommand.setWhere(
|
||||
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
metricCommand.setLimit(1000L);
|
||||
List<ColumnOrder> orders = new ArrayList<>();
|
||||
orders.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
metricCommand.setOrder(orders);
|
||||
System.out.println(parser(semanticSchema, metricCommand, true));
|
||||
|
||||
addDepartment(semanticSchema);
|
||||
|
||||
OntologyQuery metricCommand2 = new OntologyQuery();
|
||||
metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
|
||||
"user_name__department", "user_name", "user_name__page")));
|
||||
metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
|
||||
metricCommand2.setWhere(
|
||||
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
metricCommand2.setLimit(1000L);
|
||||
List<ColumnOrder> orders2 = new ArrayList<>();
|
||||
orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
metricCommand2.setOrder(orders2);
|
||||
System.out.println(parser(semanticSchema, metricCommand2, true));
|
||||
}
|
||||
|
||||
private static void addDepartment(S2CalciteSchema semanticSchema) {
|
||||
DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
datasource.setName("user_department");
|
||||
datasource.setSourceId(1L);
|
||||
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
|
||||
|
||||
MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
measure.setAgg("count");
|
||||
measure.setName("user_department_internal_cnt");
|
||||
measure.setCreateMetric("true");
|
||||
measure.setExpr("1");
|
||||
List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
measures.add(measure);
|
||||
|
||||
datasource.setMeasures(measures);
|
||||
|
||||
DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
dimension.setName("sys_imp_date");
|
||||
dimension.setExpr("imp_date");
|
||||
dimension.setType("time");
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParams.setIsPrimary("true");
|
||||
dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
dimensions.add(dimension);
|
||||
|
||||
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
dimension3.setName("sys_imp_week");
|
||||
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
dimension3.setType("time");
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
dimensionTimeTypeParams3.setTimeGranularity("week");
|
||||
dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
dimensions.add(dimension3);
|
||||
|
||||
datasource.setDimensions(dimensions);
|
||||
|
||||
List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
identify.setName("user_name");
|
||||
identify.setType("primary");
|
||||
identifies.add(identify);
|
||||
datasource.setIdentifiers(identifies);
|
||||
|
||||
semanticSchema.getDataModels().put("user_department",
|
||||
SemanticSchemaManager.getDataModel(datasource));
|
||||
|
||||
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
dimension1.setExpr("department");
|
||||
dimension1.setName("department");
|
||||
dimension1.setType("categorical");
|
||||
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
dimensionYamlTpls.add(dimension1);
|
||||
|
||||
semanticSchema.getDimensions().put("user_department",
|
||||
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
}
|
||||
//
|
||||
// public static SqlParserResp parser(S2CalciteSchema semanticSchema, OntologyQuery
|
||||
// ontologyQuery,
|
||||
// boolean isAgg) {
|
||||
// SqlParserResp sqlParser = new SqlParserResp();
|
||||
// try {
|
||||
// if (semanticSchema == null) {
|
||||
// sqlParser.setErrMsg("headlessSchema not found");
|
||||
// return sqlParser;
|
||||
// }
|
||||
// SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
|
||||
// QueryStatement queryStatement = new QueryStatement();
|
||||
// queryStatement.setOntologyQuery(ontologyQuery);
|
||||
// String sql = aggBuilder.buildOntologySql(queryStatement);
|
||||
// queryStatement.setSql(sql);
|
||||
// EngineType engineType = semanticSchema.getOntology().getDatabase().getType();
|
||||
// sqlParser.setSql(aggBuilder.getSql(engineType));
|
||||
// } catch (Exception e) {
|
||||
// sqlParser.setErrMsg(e.getMessage());
|
||||
// log.error("parser error metricQueryReq[{}] error [{}]", ontologyQuery, e);
|
||||
// }
|
||||
// return sqlParser;
|
||||
// }
|
||||
//
|
||||
// public void test() throws Exception {
|
||||
//
|
||||
// DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
// datasource.setName("s2_pv_uv_statis");
|
||||
// datasource.setSourceId(1L);
|
||||
// datasource.setSqlQuery(
|
||||
// "SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
|
||||
//
|
||||
// MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
// measure.setAgg("sum");
|
||||
// measure.setName("s2_pv_uv_statis_pv");
|
||||
// measure.setExpr("pv");
|
||||
// List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
// measures.add(measure);
|
||||
//
|
||||
// MeasureYamlTpl measure2 = new MeasureYamlTpl();
|
||||
// measure2.setAgg("count");
|
||||
// measure2.setName("s2_pv_uv_statis_internal_cnt");
|
||||
// measure2.setExpr("1");
|
||||
// measure2.setCreateMetric("true");
|
||||
// measures.add(measure2);
|
||||
//
|
||||
// MeasureYamlTpl measure3 = new MeasureYamlTpl();
|
||||
// measure3.setAgg("count");
|
||||
// measure3.setName("s2_pv_uv_statis_uv");
|
||||
// measure3.setExpr("uv");
|
||||
// measure3.setCreateMetric("true");
|
||||
// measures.add(measure3);
|
||||
//
|
||||
// datasource.setMeasures(measures);
|
||||
//
|
||||
// DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
// dimension.setName("imp_date");
|
||||
// dimension.setExpr("imp_date");
|
||||
// dimension.setType("time");
|
||||
// DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
// dimensionTimeTypeParams.setIsPrimary("true");
|
||||
// dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
// dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
// List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
// dimensions.add(dimension);
|
||||
//
|
||||
// DimensionYamlTpl dimension2 = new DimensionYamlTpl();
|
||||
// dimension2.setName("sys_imp_date");
|
||||
// dimension2.setExpr("imp_date");
|
||||
// dimension2.setType("time");
|
||||
// DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams();
|
||||
// dimensionTimeTypeParams2.setIsPrimary("true");
|
||||
// dimensionTimeTypeParams2.setTimeGranularity("day");
|
||||
// dimension2.setTypeParams(dimensionTimeTypeParams2);
|
||||
// dimensions.add(dimension2);
|
||||
//
|
||||
// DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
// dimension3.setName("sys_imp_week");
|
||||
// dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
// dimension3.setType("time");
|
||||
// DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
|
||||
// dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
// dimensionTimeTypeParams3.setTimeGranularity("day");
|
||||
// dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
// dimensions.add(dimension3);
|
||||
//
|
||||
// datasource.setDimensions(dimensions);
|
||||
//
|
||||
// List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
// IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
// identify.setName("user_name");
|
||||
// identify.setType("primary");
|
||||
// identifies.add(identify);
|
||||
// datasource.setIdentifiers(identifies);
|
||||
// S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
|
||||
//
|
||||
// SemanticSchemaManager.update(semanticSchema,
|
||||
// SemanticSchemaManager.getDataModel(datasource));
|
||||
//
|
||||
// DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
// dimension1.setExpr("page");
|
||||
// dimension1.setName("page");
|
||||
// dimension1.setType("categorical");
|
||||
// List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
// dimensionYamlTpls.add(dimension1);
|
||||
//
|
||||
// SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
|
||||
// SemanticSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
//
|
||||
// MetricYamlTpl metric1 = new MetricYamlTpl();
|
||||
// metric1.setName("pv");
|
||||
// metric1.setType("expr");
|
||||
// MetricTypeParamsYamlTpl metricTypeParams = new MetricTypeParamsYamlTpl();
|
||||
// List<MeasureYamlTpl> measures1 = new ArrayList<>();
|
||||
// MeasureYamlTpl measure1 = new MeasureYamlTpl();
|
||||
// measure1.setName("s2_pv_uv_statis_pv");
|
||||
// measures1.add(measure1);
|
||||
// metricTypeParams.setMeasures(measures1);
|
||||
// metricTypeParams.setExpr("s2_pv_uv_statis_pv");
|
||||
// metric1.setTypeParams(metricTypeParams);
|
||||
// List<MetricYamlTpl> metric = new ArrayList<>();
|
||||
// metric.add(metric1);
|
||||
//
|
||||
// MetricYamlTpl metric2 = new MetricYamlTpl();
|
||||
// metric2.setName("uv");
|
||||
// metric2.setType("expr");
|
||||
// MetricTypeParamsYamlTpl metricTypeParams1 = new MetricTypeParamsYamlTpl();
|
||||
// List<MeasureYamlTpl> measures2 = new ArrayList<>();
|
||||
// MeasureYamlTpl measure4 = new MeasureYamlTpl();
|
||||
// measure4.setName("s2_pv_uv_statis_uv");
|
||||
// measures2.add(measure4);
|
||||
// metricTypeParams1.setMeasures(measures2);
|
||||
// metricTypeParams1.setExpr("s2_pv_uv_statis_uv");
|
||||
// metric2.setTypeParams(metricTypeParams1);
|
||||
// metric.add(metric2);
|
||||
//
|
||||
// // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
|
||||
//
|
||||
// OntologyQuery metricCommand = new OntologyQuery();
|
||||
// metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
|
||||
// metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
|
||||
// metricCommand.setWhere(
|
||||
// "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
// metricCommand.setLimit(1000L);
|
||||
// List<ColumnOrder> orders = new ArrayList<>();
|
||||
// orders.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
// metricCommand.setOrder(orders);
|
||||
// System.out.println(parser(semanticSchema, metricCommand, true));
|
||||
//
|
||||
// addDepartment(semanticSchema);
|
||||
//
|
||||
// OntologyQuery metricCommand2 = new OntologyQuery();
|
||||
// metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
|
||||
// "user_name__department", "user_name", "user_name__page")));
|
||||
// metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
|
||||
// metricCommand2.setWhere(
|
||||
// "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
// metricCommand2.setLimit(1000L);
|
||||
// List<ColumnOrder> orders2 = new ArrayList<>();
|
||||
// orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
// metricCommand2.setOrder(orders2);
|
||||
// System.out.println(parser(semanticSchema, metricCommand2, true));
|
||||
// }
|
||||
//
|
||||
// private static void addDepartment(S2CalciteSchema semanticSchema) {
|
||||
// DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
// datasource.setName("user_department");
|
||||
// datasource.setSourceId(1L);
|
||||
// datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
|
||||
//
|
||||
// MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
// measure.setAgg("count");
|
||||
// measure.setName("user_department_internal_cnt");
|
||||
// measure.setCreateMetric("true");
|
||||
// measure.setExpr("1");
|
||||
// List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
// measures.add(measure);
|
||||
//
|
||||
// datasource.setMeasures(measures);
|
||||
//
|
||||
// DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
// dimension.setName("sys_imp_date");
|
||||
// dimension.setExpr("imp_date");
|
||||
// dimension.setType("time");
|
||||
// DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
|
||||
// dimensionTimeTypeParams.setIsPrimary("true");
|
||||
// dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
// dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
// List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
// dimensions.add(dimension);
|
||||
//
|
||||
// DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
// dimension3.setName("sys_imp_week");
|
||||
// dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
// dimension3.setType("time");
|
||||
// DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
|
||||
// dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
// dimensionTimeTypeParams3.setTimeGranularity("week");
|
||||
// dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
// dimensions.add(dimension3);
|
||||
//
|
||||
// datasource.setDimensions(dimensions);
|
||||
//
|
||||
// List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
// IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
// identify.setName("user_name");
|
||||
// identify.setType("primary");
|
||||
// identifies.add(identify);
|
||||
// datasource.setIdentifiers(identifies);
|
||||
//
|
||||
// semanticSchema.getDataModels().put("user_department",
|
||||
// SemanticSchemaManager.getDataModel(datasource));
|
||||
//
|
||||
// DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
// dimension1.setExpr("department");
|
||||
// dimension1.setName("department");
|
||||
// dimension1.setType("categorical");
|
||||
// List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
// dimensionYamlTpls.add(dimension1);
|
||||
//
|
||||
// semanticSchema.getDimensions().put("user_department",
|
||||
// SemanticSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
// }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package com.tencent.supersonic.headless.server.service;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.DataFormat;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||
import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl;
|
||||
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
|
||||
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class MetricServiceImplTest {
|
||||
//
|
||||
// @Test
|
||||
// void createMetric() throws Exception {
|
||||
// MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
|
||||
// ModelService modelService = Mockito.mock(ModelService.class);
|
||||
// MetricService metricService = mockMetricService(metricRepository, modelService);
|
||||
// MetricReq metricReq = buildMetricReq();
|
||||
// when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
|
||||
// when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
|
||||
// MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getDefaultUser());
|
||||
// MetricResp expectedMetricResp = buildExpectedMetricResp();
|
||||
// Assertions.assertEquals(expectedMetricResp, actualMetricResp);
|
||||
// }
|
||||
//
|
||||
// @Test
|
||||
// void updateMetric() throws Exception {
|
||||
// MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
|
||||
// ModelService modelService = Mockito.mock(ModelService.class);
|
||||
// MetricService metricService = mockMetricService(metricRepository, modelService);
|
||||
// MetricReq metricReq = buildMetricUpdateReq();
|
||||
// when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
|
||||
// when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
|
||||
// MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq());
|
||||
// when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO);
|
||||
// MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getDefaultUser());
|
||||
// MetricResp expectedMetricResp = buildExpectedMetricResp();
|
||||
// Assertions.assertEquals(expectedMetricResp, actualMetricResp);
|
||||
// }
|
||||
//
|
||||
// private MetricService mockMetricService(MetricRepository metricRepository,
|
||||
// ModelService modelService) {
|
||||
// AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class);
|
||||
// CollectService collectService = Mockito.mock(CollectService.class);
|
||||
// ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
|
||||
// DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
|
||||
// DimensionService dimensionService = Mockito.mock(DimensionService.class);
|
||||
// ChatLayerService chatLayerService = Mockito.mock(ChatLayerService.class);
|
||||
// return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
|
||||
// collectService, dataSetService, eventPublisher, dimensionService, chatLayerService);
|
||||
// }
|
||||
//
|
||||
// private MetricReq buildMetricReq() {
|
||||
// MetricReq metricReq = new MetricReq();
|
||||
// metricReq.setId(1L);
|
||||
// metricReq.setName("hr部门的访问次数");
|
||||
// metricReq.setBizName("pv");
|
||||
// metricReq.setDescription("SuperSonic的访问情况");
|
||||
// metricReq.setAlias("pv");
|
||||
// metricReq.setMetricDefineType(MetricDefineType.MEASURE);
|
||||
// metricReq.setModelId(2L);
|
||||
// metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
|
||||
// DataFormat dataFormat = new DataFormat();
|
||||
// dataFormat.setDecimalPlaces(3);
|
||||
// dataFormat.setNeedMultiply100(false);
|
||||
// metricReq.setDataFormat(dataFormat);
|
||||
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
|
||||
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
|
||||
// new MeasureParam("s2_uv", "department='hr'")));
|
||||
// typeParams.setExpr("s2_pv/s2_uv");
|
||||
// metricReq.setMetricDefineByMeasureParams(typeParams);
|
||||
// metricReq.setClassifications(Lists.newArrayList("核心指标"));
|
||||
// metricReq.setRelateDimension(RelateDimension.builder().drillDownDimensions(
|
||||
// Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
|
||||
// .build());
|
||||
// metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
|
||||
// metricReq.setExt(new HashMap<>());
|
||||
// return metricReq;
|
||||
// }
|
||||
//
|
||||
// private MetricResp buildExpectedMetricResp() {
|
||||
// MetricResp metricResp = new MetricResp();
|
||||
// metricResp.setId(1L);
|
||||
// metricResp.setName("hr部门的访问次数");
|
||||
// metricResp.setBizName("pv");
|
||||
// metricResp.setDescription("SuperSonic的访问情况");
|
||||
// metricResp.setAlias("pv");
|
||||
// metricResp.setMetricDefineType(MetricDefineType.MEASURE);
|
||||
// metricResp.setModelId(2L);
|
||||
// metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
|
||||
// DataFormat dataFormat = new DataFormat();
|
||||
// dataFormat.setDecimalPlaces(3);
|
||||
// dataFormat.setNeedMultiply100(false);
|
||||
// metricResp.setDataFormat(dataFormat);
|
||||
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
|
||||
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
|
||||
// new MeasureParam("s2_uv", "department='hr'")));
|
||||
// typeParams.setExpr("s2_pv/s2_uv");
|
||||
// metricResp.setMetricDefineByMeasureParams(typeParams);
|
||||
// metricResp.setClassifications("核心指标");
|
||||
// metricResp.setRelateDimension(RelateDimension.builder().drillDownDimensions(
|
||||
// Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
|
||||
// .build());
|
||||
// metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
|
||||
// metricResp.setExt(new HashMap<>());
|
||||
// metricResp.setTypeEnum(TypeEnums.METRIC);
|
||||
// metricResp.setIsCollect(false);
|
||||
// metricResp.setType(MetricType.DERIVED.name());
|
||||
// metricResp.setStatus(StatusEnum.ONLINE.getCode());
|
||||
// return metricResp;
|
||||
// }
|
||||
//
|
||||
// private MetricReq buildMetricUpdateReq() {
|
||||
// MetricReq metricReq = new MetricReq();
|
||||
// metricReq.setId(1L);
|
||||
// metricReq.setName("hr部门的访问次数");
|
||||
// metricReq.setBizName("pv");
|
||||
// metricReq.setMetricDefineType(MetricDefineType.MEASURE);
|
||||
// MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
|
||||
// typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
|
||||
// new MeasureParam("s2_uv", "department='hr'")));
|
||||
// typeParams.setExpr("s2_pv/s2_uv");
|
||||
// metricReq.setMetricDefineByMeasureParams(typeParams);
|
||||
// return metricReq;
|
||||
// }
|
||||
//
|
||||
// private ModelResp mockModelResp() {
|
||||
// ModelResp modelResp = new ModelResp();
|
||||
// modelResp.setId(2L);
|
||||
// modelResp.setDomainId(1L);
|
||||
// return modelResp;
|
||||
// }
|
||||
}
|
||||
@@ -22,10 +22,10 @@ import com.tencent.supersonic.headless.server.utils.ModelConverter;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -77,10 +77,10 @@ class ModelServiceImplTest {
|
||||
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
|
||||
DataSetService viewService = Mockito.mock(DataSetService.class);
|
||||
ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class);
|
||||
ThreadPoolExecutor threadPoolExecutor = Mockito.mock(ThreadPoolExecutor.class);
|
||||
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
|
||||
return new ModelServiceImpl(modelRepository, databaseService, dimensionService,
|
||||
metricService, domainService, userService, viewService, dateInfoRepository,
|
||||
modelRelaService, threadPoolExecutor);
|
||||
modelRelaService, eventPublisher);
|
||||
}
|
||||
|
||||
private ModelReq mockModelReq() {
|
||||
|
||||
Reference in New Issue
Block a user