mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-01-01 06:47:47 +08:00
[improvement][headless]Refactor translator module to make code logic cleaner and more readable.
This commit is contained in:
jerryjzhang
@@ -20,8 +20,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
|
||||
import com.tencent.supersonic.headless.core.cache.QueryCache;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
|
||||
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
|
||||
import com.tencent.supersonic.headless.core.pojo.SqlQuery;
|
||||
import com.tencent.supersonic.headless.core.pojo.StructQuery;
|
||||
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
|
||||
@@ -129,8 +129,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
for (QueryExecutor queryExecutor : queryExecutors) {
|
||||
if (queryExecutor.accept(queryStatement)) {
|
||||
queryResp = queryExecutor.execute(queryStatement);
|
||||
queryUtils.populateQueryColumns(queryResp,
|
||||
queryStatement.getSemanticSchemaResp());
|
||||
queryUtils.populateQueryColumns(queryResp, queryStatement.getSemanticSchema());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,9 +286,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
|
||||
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
|
||||
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
|
||||
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
|
||||
queryStatement.setIsTranslated(true);
|
||||
}
|
||||
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
|
||||
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
@@ -302,7 +302,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
|
||||
queryStatement.setDataSetId(queryReq.getDataSetId());
|
||||
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
|
||||
queryStatement.setDataSetName(queryReq.getDataSetName());
|
||||
queryStatement.setSemanticSchema(semanticSchemaResp);
|
||||
queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp));
|
||||
return queryStatement;
|
||||
}
|
||||
@@ -311,9 +312,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
QueryStatement queryStatement = buildQueryStatement(querySqlReq);
|
||||
queryStatement.setIsS2SQL(true);
|
||||
|
||||
SqlQueryParam sqlQueryParam = new SqlQueryParam();
|
||||
sqlQueryParam.setSql(querySqlReq.getSql());
|
||||
queryStatement.setSqlQueryParam(sqlQueryParam);
|
||||
SqlQuery sqlQuery = new SqlQuery();
|
||||
sqlQuery.setSql(querySqlReq.getSql());
|
||||
queryStatement.setSqlQuery(sqlQuery);
|
||||
|
||||
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
|
||||
if (querySqlReq.needGetDataSetId()) {
|
||||
@@ -325,9 +326,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
|
||||
private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) {
|
||||
QueryStatement queryStatement = buildQueryStatement(queryReq);
|
||||
StructQueryParam structQueryParam = new StructQueryParam();
|
||||
BeanUtils.copyProperties(queryReq, structQueryParam);
|
||||
queryStatement.setStructQueryParam(structQueryParam);
|
||||
StructQuery structQuery = new StructQuery();
|
||||
BeanUtils.copyProperties(queryReq, structQuery);
|
||||
queryStatement.setStructQuery(structQuery);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
package com.tencent.supersonic.headless.server.manager;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.headless.api.pojo.FieldParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.*;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -47,7 +38,7 @@ public class MetricYamlManager {
|
||||
MetricDefineByMeasureParams metricDefineParams =
|
||||
metric.getMetricDefineByMeasureParams();
|
||||
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
|
||||
List<MeasureParam> measures = metricDefineParams.getMeasures();
|
||||
List<Measure> measures = metricDefineParams.getMeasures();
|
||||
metricTypeParamsYamlTpl.setMeasures(
|
||||
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
||||
} else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) {
|
||||
@@ -68,7 +59,7 @@ public class MetricYamlManager {
|
||||
return metricYamlTpl;
|
||||
}
|
||||
|
||||
public static MeasureYamlTpl convert(MeasureParam measure) {
|
||||
public static MeasureYamlTpl convert(Measure measure) {
|
||||
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
|
||||
measureYamlTpl.setName(measure.getBizName());
|
||||
measureYamlTpl.setConstraint(measure.getConstraint());
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
package com.tencent.supersonic.headless.server.manager;
|
||||
|
||||
import com.tencent.supersonic.headless.api.pojo.Dimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.Identify;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
@@ -52,8 +49,7 @@ public class ModelYamlManager {
|
||||
dimensionYamlTpl.setExpr(dim.getBizName());
|
||||
}
|
||||
if (dim.getTypeParams() != null) {
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl =
|
||||
new DimensionTimeTypeParamsTpl();
|
||||
DimensionTimeTypeParams dimensionTimeTypeParamsTpl = new DimensionTimeTypeParams();
|
||||
dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary());
|
||||
dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity());
|
||||
dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl);
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
package com.tencent.supersonic.headless.server.manager;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.DataModel;
|
||||
import com.tencent.supersonic.headless.core.pojo.JoinRelation;
|
||||
import com.tencent.supersonic.headless.core.pojo.Ontology;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.s2sql.*;
|
||||
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType;
|
||||
@@ -31,6 +41,16 @@ public class SemanticSchemaManager {
|
||||
|
||||
public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) {
|
||||
Ontology ontology = new Ontology();
|
||||
ontology.setMetrics(semanticSchemaResp.getMetrics());
|
||||
Map<String, List<DimSchemaResp>> model2Dimensions = Maps.newHashMap();
|
||||
semanticSchemaResp.getDimensions().forEach(dim -> {
|
||||
if (!model2Dimensions.containsKey(dim.getModelBizName())) {
|
||||
model2Dimensions.put(dim.getModelBizName(), Lists.newArrayList());
|
||||
}
|
||||
model2Dimensions.get(dim.getModelBizName()).add(dim);
|
||||
});
|
||||
ontology.setDimensionMap(model2Dimensions);
|
||||
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
|
||||
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
|
||||
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
|
||||
@@ -49,24 +69,15 @@ public class SemanticSchemaManager {
|
||||
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
|
||||
ontology.setDataModelMap(dataModelMap);
|
||||
}
|
||||
if (!dimensionYamlTpls.isEmpty()) {
|
||||
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
|
||||
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
|
||||
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
|
||||
}
|
||||
ontology.setDimensionMap(dimensionMap);
|
||||
}
|
||||
if (!metricYamlTpls.isEmpty()) {
|
||||
ontology.setMetrics(getMetrics(metricYamlTpls));
|
||||
}
|
||||
|
||||
return ontology;
|
||||
}
|
||||
|
||||
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
||||
public static List<MetricSchemaResp> getMetrics(final List<MetricYamlTpl> t) {
|
||||
return getMetricsByMetricYamlTpl(t);
|
||||
}
|
||||
|
||||
public static List<Dimension> getDimensions(final List<DimensionYamlTpl> t) {
|
||||
public static List<DimSchemaResp> getDimensions(final List<DimensionYamlTpl> t) {
|
||||
return getDimension(t);
|
||||
}
|
||||
|
||||
@@ -83,23 +94,21 @@ public class SemanticSchemaManager {
|
||||
return dataModel;
|
||||
}
|
||||
|
||||
private static String getDataModelAggTime(List<Dimension> dimensions) {
|
||||
Optional<Dimension> timeDimension = dimensions.stream()
|
||||
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
|
||||
.findFirst();
|
||||
if (timeDimension.isPresent()
|
||||
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
|
||||
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
|
||||
private static String getDataModelAggTime(List<DimSchemaResp> dimensions) {
|
||||
Optional<DimSchemaResp> timeDimension =
|
||||
dimensions.stream().filter(DimSchemaResp::isTimeDimension).findFirst();
|
||||
if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getTypeParams())) {
|
||||
return timeDimension.get().getTypeParams().getTimeGranularity();
|
||||
}
|
||||
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
|
||||
}
|
||||
|
||||
private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
|
||||
List<Metric> metrics = new ArrayList<>();
|
||||
private static List<MetricSchemaResp> getMetricsByMetricYamlTpl(
|
||||
List<MetricYamlTpl> metricYamlTpls) {
|
||||
List<MetricSchemaResp> metrics = new ArrayList<>();
|
||||
for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {
|
||||
Metric metric = new Metric();
|
||||
metric.setMetricTypeParams(getMetricTypeParams(metricYamlTpl.getTypeParams()));
|
||||
metric.setOwners(metricYamlTpl.getOwners());
|
||||
MetricSchemaResp metric = new MetricSchemaResp();
|
||||
fillMetricTypeParams(metric, metricYamlTpl.getTypeParams());
|
||||
metric.setType(metricYamlTpl.getType());
|
||||
metric.setName(metricYamlTpl.getName());
|
||||
metrics.add(metric);
|
||||
@@ -107,55 +116,50 @@ public class SemanticSchemaManager {
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static MetricTypeParams getMetricTypeParams(
|
||||
private static void fillMetricTypeParams(MetricSchemaResp metric,
|
||||
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
|
||||
MetricTypeParams metricTypeParams = new MetricTypeParams();
|
||||
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
metricTypeParams.setFieldMetric(false);
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
|
||||
metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
|
||||
MetricDefineByMeasureParams params = new MetricDefineByMeasureParams();
|
||||
params.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
|
||||
metric.setMetricDefinition(MetricDefineType.MEASURE, params);
|
||||
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
|
||||
MetricDefineByMetricParams params = new MetricDefineByMetricParams();
|
||||
params.setMetrics(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
|
||||
params.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
metric.setMetricDefinition(MetricDefineType.METRIC, params);
|
||||
} else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
|
||||
MetricDefineByFieldParams params = new MetricDefineByFieldParams();
|
||||
params.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
params.setFields(getFieldParams(metricTypeParamsYamlTpl.getFields()));
|
||||
metric.setMetricDefinition(MetricDefineType.FIELD, params);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
|
||||
metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
|
||||
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
|
||||
metricTypeParams.setFieldMetric(true);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
|
||||
metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
|
||||
metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
|
||||
metricTypeParams.setFieldMetric(true);
|
||||
}
|
||||
|
||||
return metricTypeParams;
|
||||
}
|
||||
|
||||
private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
private static List<FieldParam> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
|
||||
List<FieldParam> fields = new ArrayList<>();
|
||||
for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setName(fieldParamYamlTpl.getFieldName());
|
||||
measure.setExpr(fieldParamYamlTpl.getFieldName());
|
||||
measures.add(measure);
|
||||
FieldParam field = new FieldParam();
|
||||
field.setFieldName(fieldParamYamlTpl.getFieldName());
|
||||
fields.add(field);
|
||||
}
|
||||
return measures;
|
||||
return fields;
|
||||
}
|
||||
|
||||
private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
private static List<MetricParam> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
|
||||
List<MetricParam> metrics = new ArrayList<>();
|
||||
for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setName(metricParamYamlTpl.getBizName());
|
||||
measure.setExpr(metricParamYamlTpl.getBizName());
|
||||
measures.add(measure);
|
||||
MetricParam metric = new MetricParam();
|
||||
metric.setBizName(metricParamYamlTpl.getBizName());
|
||||
metric.setId(metricParamYamlTpl.getId());
|
||||
metrics.add(metric);
|
||||
}
|
||||
return measures;
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setCreateMetric(measureYamlTpl.getCreateMetric());
|
||||
measure.setExpr(measureYamlTpl.getExpr());
|
||||
measure.setAgg(measureYamlTpl.getAgg());
|
||||
measure.setName(measureYamlTpl.getName());
|
||||
@@ -166,34 +170,32 @@ public class SemanticSchemaManager {
|
||||
return measures;
|
||||
}
|
||||
|
||||
private static List<Dimension> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
private static List<DimSchemaResp> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
|
||||
List<DimSchemaResp> dimensions = new ArrayList<>();
|
||||
for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) {
|
||||
Dimension dimension = Dimension.builder().build();
|
||||
dimension.setType(dimensionYamlTpl.getType());
|
||||
DimSchemaResp dimension = new DimSchemaResp();
|
||||
// dimension.setType(dimensionYamlTpl.getType());
|
||||
dimension.setExpr(dimensionYamlTpl.getExpr());
|
||||
dimension.setName(dimensionYamlTpl.getName());
|
||||
dimension.setOwners(dimensionYamlTpl.getOwners());
|
||||
dimension.setBizName(dimensionYamlTpl.getBizName());
|
||||
dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
|
||||
if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
|
||||
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
|
||||
dimension.setDataType(dimensionYamlTpl.getDataType());
|
||||
}
|
||||
if (Objects.isNull(dimension.getDataType())) {
|
||||
dimension.setDataType(DataType.UNKNOWN);
|
||||
dimension.setDataType(DataTypeEnums.UNKNOWN);
|
||||
}
|
||||
if (Objects.nonNull(dimensionYamlTpl.getExt())) {
|
||||
dimension.setExt(dimensionYamlTpl.getExt());
|
||||
}
|
||||
dimension.setDimensionTimeTypeParams(
|
||||
getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()));
|
||||
dimension.setTypeParams(dimensionYamlTpl.getTypeParams());
|
||||
dimensions.add(dimension);
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
private static DimensionTimeTypeParams getDimensionTimeTypeParams(
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
|
||||
DimensionTimeTypeParams dimensionTimeTypeParamsTpl) {
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
if (dimensionTimeTypeParamsTpl != null) {
|
||||
dimensionTimeTypeParams
|
||||
@@ -238,7 +240,8 @@ public class SemanticSchemaManager {
|
||||
return joinRelations;
|
||||
}
|
||||
|
||||
public static void update(S2CalciteSchema schema, List<Metric> metric) throws Exception {
|
||||
public static void update(S2CalciteSchema schema, List<MetricSchemaResp> metric)
|
||||
throws Exception {
|
||||
if (schema != null) {
|
||||
updateMetric(metric, schema.getMetrics());
|
||||
}
|
||||
@@ -260,31 +263,31 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
|
||||
public static void update(S2CalciteSchema schema, String datasourceBizName,
|
||||
List<Dimension> dimensionYamlTpls) throws Exception {
|
||||
List<DimSchemaResp> dimensionYamlTpls) throws Exception {
|
||||
if (schema != null) {
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
|
||||
Optional<Map.Entry<String, List<DimSchemaResp>>> datasourceYamlTplMap = schema
|
||||
.getDimensions().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
|
||||
} else {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
List<DimSchemaResp> dimensions = new ArrayList<>();
|
||||
updateDimension(dimensionYamlTpls, dimensions);
|
||||
schema.getDimensions().put(datasourceBizName, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateDimension(List<Dimension> dimensionYamlTpls,
|
||||
List<Dimension> dimensions) {
|
||||
private static void updateDimension(List<DimSchemaResp> dimensionYamlTpls,
|
||||
List<DimSchemaResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd =
|
||||
dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Dimension> iterator = dimensions.iterator();
|
||||
Iterator<DimSchemaResp> iterator = dimensions.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Dimension cur = iterator.next();
|
||||
DimSchemaResp cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
@@ -292,15 +295,16 @@ public class SemanticSchemaManager {
|
||||
dimensions.addAll(dimensionYamlTpls);
|
||||
}
|
||||
|
||||
private static void updateMetric(List<Metric> metricYamlTpls, List<Metric> metrics) {
|
||||
private static void updateMetric(List<MetricSchemaResp> metricYamlTpls,
|
||||
List<MetricSchemaResp> metrics) {
|
||||
if (CollectionUtils.isEmpty(metricYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd =
|
||||
metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Metric> iterator = metrics.iterator();
|
||||
Iterator<MetricSchemaResp> iterator = metrics.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Metric cur = iterator.next();
|
||||
MetricSchemaResp cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
package com.tencent.supersonic.headless.server.pojo.yaml;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class DimensionTimeTypeParamsTpl {
|
||||
|
||||
private String isPrimary;
|
||||
|
||||
private String timeGranularity;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.tencent.supersonic.headless.server.pojo.yaml;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
@@ -19,7 +20,7 @@ public class DimensionYamlTpl {
|
||||
|
||||
private String expr;
|
||||
|
||||
private DimensionTimeTypeParamsTpl typeParams;
|
||||
private DimensionTimeTypeParams typeParams;
|
||||
|
||||
private DataTypeEnums dataType;
|
||||
|
||||
|
||||
@@ -8,58 +8,22 @@ import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataItem;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.common.pojo.*;
|
||||
import com.tencent.supersonic.common.pojo.enums.*;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
||||
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||
import com.tencent.supersonic.headless.server.service.CollectService;
|
||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.pojo.*;
|
||||
import com.tencent.supersonic.headless.server.service.*;
|
||||
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
@@ -72,18 +36,7 @@ import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@@ -427,8 +380,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
return true;
|
||||
}
|
||||
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
|
||||
List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
|
||||
List<Measure> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
List<String> fieldNameDepended = measures.stream().map(Measure::getName)
|
||||
// measure bizName = model bizName_fieldName
|
||||
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
|
||||
.collect(Collectors.toList());
|
||||
@@ -705,12 +658,11 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
// Measure define will get from first measure
|
||||
if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
|
||||
List<Measure> measures = modelResp.getModelDetail().getMeasures();
|
||||
List<MeasureParam> measureParams =
|
||||
metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
if (CollectionUtils.isEmpty(measureParams)) {
|
||||
return null;
|
||||
}
|
||||
MeasureParam firstMeasure = measureParams.get(0);
|
||||
Measure firstMeasure = measureParams.get(0);
|
||||
|
||||
for (Measure measure : measures) {
|
||||
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
|
||||
|
||||
@@ -26,7 +26,7 @@ public class MetricDrillDownChecker {
|
||||
private MetricService metricService;
|
||||
|
||||
public void checkQuery(QueryStatement queryStatement) {
|
||||
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
|
||||
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema();
|
||||
String sql = queryStatement.getSql();
|
||||
if (StringUtils.isBlank(sql)) {
|
||||
return;
|
||||
|
||||
@@ -7,21 +7,8 @@ import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.Identify;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
|
||||
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
|
||||
import com.tencent.supersonic.headless.api.pojo.ModelSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
|
||||
@@ -34,12 +21,7 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ModelConverter {
|
||||
@@ -141,9 +123,7 @@ public class ModelConverter {
|
||||
metricReq.setModelId(modelDO.getId());
|
||||
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
|
||||
exprTypeParams.setExpr(measure.getBizName());
|
||||
MeasureParam measureParam = new MeasureParam();
|
||||
BeanMapper.mapper(measure, measureParam);
|
||||
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
|
||||
exprTypeParams.setMeasures(Lists.newArrayList(measure));
|
||||
metricReq.setMetricDefineByMeasureParams(exprTypeParams);
|
||||
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
|
||||
return metricReq;
|
||||
|
||||
Reference in New Issue
Block a user