[improvement][headless]Release brand new version of Translator module.

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
 import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.MetaFilter;
 import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
 import com.tencent.supersonic.headless.api.pojo.request.*;
@@ -30,6 +31,7 @@ import com.tencent.supersonic.headless.server.service.DataSetService;
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.SchemaService;
+import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
 import com.tencent.supersonic.headless.server.utils.StatUtils;
 import lombok.SneakyThrows;
@@ -52,6 +54,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
     private final DataSetService dataSetService;
     private final SchemaService schemaService;
     private final SemanticTranslator semanticTranslator;
+    private final MetricDrillDownChecker metricDrillDownChecker;
     private final KnowledgeBaseService knowledgeBaseService;
     private final MetricService metricService;
     private final DimensionService dimensionService;
@@ -61,6 +64,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
     public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils,
             SemanticSchemaManager semanticSchemaManager, DataSetService dataSetService,
             SchemaService schemaService, SemanticTranslator semanticTranslator,
+            MetricDrillDownChecker metricDrillDownChecker,
             KnowledgeBaseService knowledgeBaseService, MetricService metricService,
             DimensionService dimensionService) {
         this.statUtils = statUtils;
@@ -69,6 +73,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
         this.dataSetService = dataSetService;
         this.schemaService = schemaService;
         this.semanticTranslator = semanticTranslator;
+        this.metricDrillDownChecker = metricDrillDownChecker;
         this.knowledgeBaseService = knowledgeBaseService;
         this.metricService = metricService;
         this.dimensionService = dimensionService;
@@ -115,6 +120,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
             QueryStatement queryStatement = buildQueryStatement(queryReq, user);
             semanticTranslator.translate(queryStatement);
 
+            // Check whether the dimensions of the metric drill-down are correct temporarily,
+            // add the abstraction of a validator later.
+            metricDrillDownChecker.checkQuery(queryStatement);
+
             // 4.execute query
             SemanticQueryResp queryResp = null;
             for (QueryExecutor queryExecutor : queryExecutors) {
@@ -198,6 +207,14 @@ public class S2SemanticLayerService implements SemanticLayerService {
         ModelResp modelResp = modelResps.get(0);
         String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
                 modelResp.getName());
+        List<Dimension> timeDims = modelResp.getTimeDimension();
+        if (CollectionUtils.isNotEmpty(timeDims)) {
+            sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
+                    queryDimValueReq.getDateInfo().getDateField(),
+                    queryDimValueReq.getDateInfo().getStartDate(),
+                    queryDimValueReq.getDateInfo().getDateField(),
+                    queryDimValueReq.getDateInfo().getEndDate());
+        }
         if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
             sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
                     + queryDimValueReq.getValue() + "%'";
@@ -269,10 +286,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
         if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
                 && StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
             queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
-            queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
-            queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
             queryStatement.setIsTranslated(true);
         }
+        queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
+        queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
         return queryStatement;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -4,10 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
+import com.tencent.supersonic.headless.server.pojo.yaml.*;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -1,14 +1,16 @@
 package com.tencent.supersonic.headless.server.manager;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.tencent.supersonic.common.pojo.ModelRela;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.*;
+import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
+import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.core.pojo.JoinRelation;
 import com.tencent.supersonic.headless.core.pojo.Ontology;
 import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
-import com.tencent.supersonic.headless.core.translator.parser.s2sql.*;
-import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType;
 import com.tencent.supersonic.headless.server.pojo.yaml.*;
 import com.tencent.supersonic.headless.server.service.SchemaService;
 import lombok.extern.slf4j.Slf4j;
@@ -32,6 +34,24 @@ public class SemanticSchemaManager {
 
     public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) {
         Ontology ontology = new Ontology();
+        Map<String, List<MetricSchemaResp>> model2Metrics = Maps.newHashMap();
+        semanticSchemaResp.getMetrics().forEach(dim -> {
+            if (!model2Metrics.containsKey(dim.getModelBizName())) {
+                model2Metrics.put(dim.getModelBizName(), Lists.newArrayList());
+            }
+            model2Metrics.get(dim.getModelBizName()).add(dim);
+        });
+        ontology.setMetricMap(model2Metrics);
+
+        Map<String, List<DimSchemaResp>> model2Dimensions = Maps.newHashMap();
+        semanticSchemaResp.getDimensions().forEach(dim -> {
+            if (!model2Dimensions.containsKey(dim.getModelBizName())) {
+                model2Dimensions.put(dim.getModelBizName(), Lists.newArrayList());
+            }
+            model2Dimensions.get(dim.getModelBizName()).add(dim);
+        });
+        ontology.setDimensionMap(model2Dimensions);
+
         Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
         List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
         List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
@@ -45,25 +65,16 @@ public class SemanticSchemaManager {
                     getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
         }
         if (!dataModelYamlTpls.isEmpty()) {
-            Map<String, DataModel> dataModelMap =
+            Map<String, ModelResp> dataModelMap =
                     dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect(
-                            Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
-            ontology.setDataModelMap(dataModelMap);
-        }
-        if (!dimensionYamlTpls.isEmpty()) {
-            Map<String, List<Dimension>> dimensionMap = new HashMap<>();
-            for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
-                dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
-            }
-            ontology.setDimensionMap(dimensionMap);
-        }
-        if (!metricYamlTpls.isEmpty()) {
-            ontology.setMetrics(getMetrics(metricYamlTpls));
+                            Collectors.toMap(ModelResp::getName, item -> item, (k1, k2) -> k1));
+            ontology.setModelMap(dataModelMap);
         }
+
         return ontology;
     }
 
-    public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
+    public static List<MetricSchemaResp> getMetrics(final List<MetricYamlTpl> t) {
         return getMetricsByMetricYamlTpl(t);
     }
 
@@ -71,36 +82,34 @@ public class SemanticSchemaManager {
         return getDimension(t);
     }
 
-    public static DataModel getDataModel(final DataModelYamlTpl d) {
-        DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
-                .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
-                .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
-                .measures(getMeasureParams(d.getMeasures()))
-                .dimensions(getDimensions(d.getDimensions())).build();
-        dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions()));
-        if (Objects.nonNull(d.getModelSourceTypeEnum())) {
-            dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
-        }
+    public static ModelResp getDataModel(final DataModelYamlTpl d) {
+        // ModelResp dataModel = ModelResp.builder()(d.getId()).modelId(d.getSourceId())
+        // .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
+        // .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
+        // .measures(getMeasureParams(d.getMeasures()))
+        // .dimensions(getDimensions(d.getDimensions())).build();
+        ModelResp dataModel = new ModelResp();
+        dataModel.setId(d.getId());
+        dataModel.setName(d.getName());
+        ModelDetail modelDetail = new ModelDetail();
+        dataModel.setModelDetail(modelDetail);
+
+        modelDetail.setDbType(d.getType());
+        modelDetail.setSqlQuery(d.getSqlQuery());
+        modelDetail.setTableQuery(d.getTableQuery());
+        modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
+        modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
+        modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
+
         return dataModel;
     }
 
-    private static String getDataModelAggTime(List<Dimension> dimensions) {
-        Optional<Dimension> timeDimension = dimensions.stream()
-                .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
-                .findFirst();
-        if (timeDimension.isPresent()
-                && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
-            return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
-        }
-        return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
-    }
-
-    private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
-        List<Metric> metrics = new ArrayList<>();
+    private static List<MetricSchemaResp> getMetricsByMetricYamlTpl(
+            List<MetricYamlTpl> metricYamlTpls) {
+        List<MetricSchemaResp> metrics = new ArrayList<>();
         for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {
-            Metric metric = new Metric();
-            metric.setMetricTypeParams(getMetricTypeParams(metricYamlTpl.getTypeParams()));
-            metric.setOwners(metricYamlTpl.getOwners());
+            MetricSchemaResp metric = new MetricSchemaResp();
+            fillMetricTypeParams(metric, metricYamlTpl.getTypeParams());
             metric.setType(metricYamlTpl.getType());
             metric.setName(metricYamlTpl.getName());
             metrics.add(metric);
@@ -108,55 +117,50 @@ public class SemanticSchemaManager {
         return metrics;
     }
 
-    private static MetricTypeParams getMetricTypeParams(
+    private static void fillMetricTypeParams(MetricSchemaResp metric,
             MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
-        MetricTypeParams metricTypeParams = new MetricTypeParams();
-        metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
-        metricTypeParams.setFieldMetric(false);
         if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
-            metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
+            MetricDefineByMeasureParams params = new MetricDefineByMeasureParams();
+            params.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
+            metric.setMetricDefinition(MetricDefineType.MEASURE, params);
+        } else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
+            MetricDefineByMetricParams params = new MetricDefineByMetricParams();
+            params.setMetrics(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
+            params.setExpr(metricTypeParamsYamlTpl.getExpr());
+            metric.setMetricDefinition(MetricDefineType.METRIC, params);
+        } else if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
+            MetricDefineByFieldParams params = new MetricDefineByFieldParams();
+            params.setExpr(metricTypeParamsYamlTpl.getExpr());
+            params.setFields(getFieldParams(metricTypeParamsYamlTpl.getFields()));
+            metric.setMetricDefinition(MetricDefineType.FIELD, params);
         }
-        if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
-            metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
-            metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
-            metricTypeParams.setFieldMetric(true);
-        }
-        if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
-            metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
-            metricTypeParams.setExpr(metricTypeParams.getMeasures().get(0).getExpr());
-            metricTypeParams.setFieldMetric(true);
-        }
-
-        return metricTypeParams;
     }
 
-    private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
-        List<Measure> measures = new ArrayList<>();
+    private static List<FieldParam> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
+        List<FieldParam> fields = new ArrayList<>();
         for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
-            Measure measure = new Measure();
-            measure.setName(fieldParamYamlTpl.getFieldName());
-            measure.setExpr(fieldParamYamlTpl.getFieldName());
-            measures.add(measure);
+            FieldParam field = new FieldParam();
+            field.setFieldName(fieldParamYamlTpl.getFieldName());
+            fields.add(field);
         }
-        return measures;
+        return fields;
     }
 
-    private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
-        List<Measure> measures = new ArrayList<>();
+    private static List<MetricParam> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
+        List<MetricParam> metrics = new ArrayList<>();
         for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
-            Measure measure = new Measure();
-            measure.setName(metricParamYamlTpl.getBizName());
-            measure.setExpr(metricParamYamlTpl.getBizName());
-            measures.add(measure);
+            MetricParam metric = new MetricParam();
+            metric.setBizName(metricParamYamlTpl.getBizName());
+            metric.setId(metricParamYamlTpl.getId());
+            metrics.add(metric);
         }
-        return measures;
+        return metrics;
     }
 
     private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
         List<Measure> measures = new ArrayList<>();
         for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
             Measure measure = new Measure();
-            measure.setCreateMetric(measureYamlTpl.getCreateMetric());
             measure.setExpr(measureYamlTpl.getExpr());
             measure.setAgg(measureYamlTpl.getAgg());
             measure.setName(measureYamlTpl.getName());
@@ -170,28 +174,30 @@ public class SemanticSchemaManager {
     private static List<Dimension> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
         List<Dimension> dimensions = new ArrayList<>();
         for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) {
-            Dimension dimension = Dimension.builder().build();
-            dimension.setType(dimensionYamlTpl.getType());
+            Dimension dimension = new Dimension();
+            if (Objects.nonNull(dimensionYamlTpl.getType())) {
+                dimension.setType(DimensionType.valueOf(dimensionYamlTpl.getType()));
+            }
             dimension.setExpr(dimensionYamlTpl.getExpr());
             dimension.setName(dimensionYamlTpl.getName());
-            dimension.setOwners(dimensionYamlTpl.getOwners());
             dimension.setBizName(dimensionYamlTpl.getBizName());
-            dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
-            if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
-                dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
-            }
-            if (Objects.isNull(dimension.getDataType())) {
-                dimension.setDataType(DataType.UNKNOWN);
-            }
-            if (Objects.nonNull(dimensionYamlTpl.getExt())) {
-                dimension.setExt(dimensionYamlTpl.getExt());
-            }
-            dimension.setDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams());
+            dimension.setTypeParams(dimensionYamlTpl.getTypeParams());
             dimensions.add(dimension);
         }
         return dimensions;
     }
 
+    private static DimensionTimeTypeParams getDimensionTimeTypeParams(
+            DimensionTimeTypeParams dimensionTimeTypeParamsTpl) {
+        DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
+        if (dimensionTimeTypeParamsTpl != null) {
+            dimensionTimeTypeParams
+                    .setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
+            dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
+        }
+        return dimensionTimeTypeParams;
+    }
+
     private static List<Identify> getIdentify(List<IdentifyYamlTpl> identifyYamlTpls) {
         List<Identify> identifies = new ArrayList<>();
         for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) {
@@ -227,17 +233,18 @@ public class SemanticSchemaManager {
         return joinRelations;
     }
 
-    public static void update(S2CalciteSchema schema, List<Metric> metric) throws Exception {
+    public static void update(S2CalciteSchema schema, List<MetricSchemaResp> metric)
+            throws Exception {
         if (schema != null) {
             updateMetric(metric, schema.getMetrics());
         }
     }
 
-    public static void update(S2CalciteSchema schema, DataModel datasourceYamlTpl)
+    public static void update(S2CalciteSchema schema, ModelResp datasourceYamlTpl)
             throws Exception {
         if (schema != null) {
             String dataSourceName = datasourceYamlTpl.getName();
-            Optional<Entry<String, DataModel>> datasourceYamlTplMap =
+            Optional<Entry<String, ModelResp>> datasourceYamlTplMap =
                     schema.getDataModels().entrySet().stream()
                             .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
             if (datasourceYamlTplMap.isPresent()) {
@@ -249,31 +256,31 @@ public class SemanticSchemaManager {
     }
 
     public static void update(S2CalciteSchema schema, String datasourceBizName,
-            List<Dimension> dimensionYamlTpls) throws Exception {
+            List<DimSchemaResp> dimensionYamlTpls) throws Exception {
         if (schema != null) {
-            Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
+            Optional<Map.Entry<String, List<DimSchemaResp>>> datasourceYamlTplMap = schema
                     .getDimensions().entrySet().stream()
                     .filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
             if (datasourceYamlTplMap.isPresent()) {
                 updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
             } else {
-                List<Dimension> dimensions = new ArrayList<>();
+                List<DimSchemaResp> dimensions = new ArrayList<>();
                 updateDimension(dimensionYamlTpls, dimensions);
                 schema.getDimensions().put(datasourceBizName, dimensions);
             }
         }
     }
 
-    private static void updateDimension(List<Dimension> dimensionYamlTpls,
-            List<Dimension> dimensions) {
+    private static void updateDimension(List<DimSchemaResp> dimensionYamlTpls,
+            List<DimSchemaResp> dimensions) {
         if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
             return;
         }
         Set<String> toAdd =
                 dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
-        Iterator<Dimension> iterator = dimensions.iterator();
+        Iterator<DimSchemaResp> iterator = dimensions.iterator();
         while (iterator.hasNext()) {
-            Dimension cur = iterator.next();
+            DimSchemaResp cur = iterator.next();
             if (toAdd.contains(cur.getName())) {
                 iterator.remove();
             }
@@ -281,15 +288,16 @@ public class SemanticSchemaManager {
         dimensions.addAll(dimensionYamlTpls);
     }
 
-    private static void updateMetric(List<Metric> metricYamlTpls, List<Metric> metrics) {
+    private static void updateMetric(List<MetricSchemaResp> metricYamlTpls,
+            List<MetricSchemaResp> metrics) {
         if (CollectionUtils.isEmpty(metricYamlTpls)) {
             return;
         }
         Set<String> toAdd =
                 metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
-        Iterator<Metric> iterator = metrics.iterator();
+        Iterator<MetricSchemaResp> iterator = metrics.iterator();
         while (iterator.hasNext()) {
-            Metric cur = iterator.next();
+            MetricSchemaResp cur = iterator.next();
             if (toAdd.contains(cur.getName())) {
                 iterator.remove();
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java

@@ -15,6 +15,7 @@ public class QueryStatDO {
     private String traceId;
     private Long modelId;
     private Long dataSetId;
+    @TableField("query_user")
     private String queryUser;
     private String createdAt;
     /** corresponding type, such as sql, struct, etc */
@@ -27,7 +28,8 @@ public class QueryStatDO {
     private String queryStructCmd;
     @TableField("struct_cmd_md5")
     private String queryStructCmdMd5;
-    private String querySql;
+    @TableField("query_sql")
+    private String sql;
     private String sqlMd5;
     private String queryEngine;
     // private Long startTime;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -190,7 +190,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
     private SemanticQueryResp queryWithColumns(String sql, DatabaseResp database) {
         SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
         SqlUtils sqlUtils = this.sqlUtils.init(database);
-        log.info("query SQL: {}", sql);
+        log.info("query SQL: {}", StringUtils.normalizeSpace(sql));
         sqlUtils.queryInternal(sql, queryResultWithColumns);
         return queryResultWithColumns;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -62,22 +62,19 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
 
     private DataSetService dataSetService;
 
-    private TagMetaService tagMetaService;
 
     @Autowired
     private ApplicationEventPublisher eventPublisher;
 
     public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
             AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
-            ModelRelaService modelRelaService, DataSetService dataSetService,
-            TagMetaService tagMetaService) {
+            ModelRelaService modelRelaService, DataSetService dataSetService) {
         this.modelService = modelService;
         this.dimensionRepository = dimensionRepository;
         this.aliasGenerateHelper = aliasGenerateHelper;
         this.databaseService = databaseService;
         this.modelRelaService = modelRelaService;
         this.dataSetService = dataSetService;
-        this.tagMetaService = tagMetaService;
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java

@@ -240,7 +240,7 @@ public class DownloadServiceImpl implements DownloadService {
         QueryStructReq queryStructReq = new QueryStructReq();
         queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
                 .collect(Collectors.toList()));
-        queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
+        queryStructReq.getGroups().add(0, dateConf.getDateField());
         Aggregator aggregator = new Aggregator();
         aggregator.setColumn(metricResp.getBizName());
         queryStructReq.setAggregators(Lists.newArrayList(aggregator));
@@ -250,10 +250,6 @@ public class DownloadServiceImpl implements DownloadService {
         return queryStructReq;
     }
 
-    private String getTimeDimension(DateConf dateConf) {
-        return dateConf.getDateField();
-    }
-
     private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {
         for (MetricResp metricResp : metricResps) {
             List<DrillDownDimension> drillDownDimensions =

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -8,7 +8,6 @@ import com.github.pagehelper.PageInfo;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
 import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.*;
 import com.tencent.supersonic.common.util.BeanMapper;
@@ -59,15 +58,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
 
     private ApplicationEventPublisher eventPublisher;
 
-    private TagMetaService tagMetaService;
-
     private ChatLayerService chatLayerService;
 
     public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
             AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
             DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
-            DimensionService dimensionService, TagMetaService tagMetaService,
-            @Lazy ChatLayerService chatLayerService) {
+            DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) {
         this.metricRepository = metricRepository;
         this.modelService = modelService;
         this.aliasGenerateHelper = aliasGenerateHelper;
@@ -75,7 +71,6 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
         this.collectService = collectService;
         this.dataSetService = dataSetService;
         this.dimensionService = dimensionService;
-        this.tagMetaService = tagMetaService;
         this.chatLayerService = chatLayerService;
     }
 
@@ -386,7 +381,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
             }
         } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
             List<Measure> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
-            List<String> fieldNameDepended = measures.stream().map(Measure::getBizName)
+            List<String> fieldNameDepended = measures.stream().map(Measure::getName)
                     // measure bizName = model bizName_fieldName
                     .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
                     .collect(Collectors.toList());
@@ -660,37 +655,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
                 && !metricResp.getDefaultAgg().isEmpty())) {
             return metricResp.getDefaultAgg();
         }
-        // FIELD define will get from expr
-        if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
-            return SqlSelectFunctionHelper.getFirstAggregateFunctions(metricResp.getExpr());
-        }
-        // METRIC define will get from first metric
-        if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
-            if (!CollectionUtils.isEmpty(metricResp.getMetricDefineByMetricParams().getMetrics())) {
-                MetricParam metricParam =
-                        metricResp.getMetricDefineByMetricParams().getMetrics().get(0);
-                MetricResp firstMetricResp =
-                        getMetric(modelResp.getDomainId(), metricParam.getBizName());
-                if (Objects.nonNull(firstMetricResp)) {
-                    return getDefaultAgg(firstMetricResp, modelResp);
-                }
-                return "";
-            }
-        }
         // Measure define will get from first measure
-        List<Measure> measures = modelResp.getModelDetail().getMeasures();
-        List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
-        if (CollectionUtils.isEmpty(measureParams)) {
-            return "";
-        }
-        Measure firstMeasure = measureParams.get(0);
+        if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
+            List<Measure> measures = modelResp.getModelDetail().getMeasures();
+            List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
+            if (CollectionUtils.isEmpty(measureParams)) {
+                return null;
+            }
+            Measure firstMeasure = measureParams.get(0);
 
-        for (Measure measure : measures) {
-            if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
-                return measure.getAgg();
+            for (Measure measure : measures) {
+                if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
+                    return measure.getAgg();
+                }
             }
         }
-        return "";
+
+        return null;
     }
 
     @Override
@@ -716,31 +697,31 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
             queryMetricReq.setDateInfo(null);
         }
         // 4. set groups
-        List<String> dimensionBizNames = dimensionResps.stream()
+        List<String> dimensionNames = dimensionResps.stream()
                 .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
                 .filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
                         || queryMetricReq.getDimensionNames().contains(entry.getBizName())
                         || queryMetricReq.getDimensionIds().contains(entry.getId()))
-                .map(SchemaItem::getBizName).collect(Collectors.toList());
+                .map(SchemaItem::getName).collect(Collectors.toList());
 
         QueryStructReq queryStructReq = new QueryStructReq();
         DateConf dateInfo = queryMetricReq.getDateInfo();
         if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) {
             queryStructReq.getGroups().add(dateInfo.getDateField());
         }
-        if (!CollectionUtils.isEmpty(dimensionBizNames)) {
-            queryStructReq.getGroups().addAll(dimensionBizNames);
+        if (!CollectionUtils.isEmpty(dimensionNames)) {
+            queryStructReq.getGroups().addAll(dimensionNames);
         }
         // 5. set aggregators
-        List<String> metricBizNames = metricResps.stream()
+        List<String> metricNames = metricResps.stream()
                 .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
-                .map(SchemaItem::getBizName).collect(Collectors.toList());
-        if (CollectionUtils.isEmpty(metricBizNames)) {
+                .map(SchemaItem::getName).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(metricNames)) {
             throw new IllegalArgumentException(
                     "Invalid input parameters, unable to obtain valid metrics");
         }
         List<Aggregator> aggregators = new ArrayList<>();
-        for (String metricBizName : metricBizNames) {
+        for (String metricBizName : metricNames) {
             Aggregator aggregator = new Aggregator();
             aggregator.setColumn(metricBizName);
             aggregators.add(aggregator);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -2,12 +2,15 @@ package com.tencent.supersonic.headless.server.service.impl;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.service.UserService;
+import com.tencent.supersonic.common.pojo.DataEvent;
+import com.tencent.supersonic.common.pojo.DataItem;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.common.pojo.ModelRela;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
+import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
@@ -51,7 +54,7 @@ import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -69,7 +72,10 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Service
@@ -94,13 +100,16 @@ public class ModelServiceImpl implements ModelService {
 
     private final ModelRelaService modelRelaService;
 
-    private final ThreadPoolExecutor executor;
+    private final ApplicationEventPublisher eventPublisher;
+
+    ExecutorService executor =
+            new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
 
     public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
             @Lazy DimensionService dimensionService, @Lazy MetricService metricService,
             DomainService domainService, UserService userService, DataSetService dataSetService,
             DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService,
-            @Qualifier("commonExecutor") ThreadPoolExecutor executor) {
+            ApplicationEventPublisher eventPublisher) {
         this.modelRepository = modelRepository;
         this.databaseService = databaseService;
         this.dimensionService = dimensionService;
@@ -110,17 +119,18 @@ public class ModelServiceImpl implements ModelService {
         this.dataSetService = dataSetService;
         this.dateInfoRepository = dateInfoRepository;
         this.modelRelaService = modelRelaService;
-        this.executor = executor;
+        this.eventPublisher = eventPublisher;
     }
 
     @Override
     @Transactional
     public ModelResp createModel(ModelReq modelReq, User user) throws Exception {
-        // checkParams(modelReq);
+        checkParams(modelReq);
         ModelDO modelDO = ModelConverter.convert(modelReq, user);
         modelRepository.createModel(modelDO);
         batchCreateDimension(modelDO, user);
         batchCreateMetric(modelDO, user);
+        sendEvent(modelDO, EventType.ADD);
         return ModelConverter.convert(modelDO);
     }
 
@@ -140,13 +150,14 @@ public class ModelServiceImpl implements ModelService {
     @Override
     @Transactional
     public ModelResp updateModel(ModelReq modelReq, User user) throws Exception {
-        // checkParams(modelReq);
+        checkParams(modelReq);
         checkRelations(modelReq);
         ModelDO modelDO = modelRepository.getModelById(modelReq.getId());
         ModelConverter.convert(modelDO, modelReq, user);
         modelRepository.updateModel(modelDO);
         batchCreateDimension(modelDO, user);
         batchCreateMetric(modelDO, user);
+        sendEvent(modelDO, EventType.UPDATE);
         return ModelConverter.convert(modelDO);
     }
 
@@ -607,4 +618,16 @@ public class ModelServiceImpl implements ModelService {
         }
         return false;
     }
+
+    private void sendEvent(ModelDO modelDO, EventType eventType) {
+        DataItem dataItem = getDataItem(modelDO);
+        eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
+    }
+
+    private DataItem getDataItem(ModelDO modelDO) {
+        return DataItem.builder().id(modelDO.getId().toString()).name(modelDO.getName())
+                .bizName(modelDO.getBizName()).modelId(modelDO.getId().toString())
+                .domainId(modelDO.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java

@@ -2,17 +2,7 @@ package com.tencent.supersonic.headless.server.utils;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.DimensionConstants;
-import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.common.util.DateUtils;
-import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
-import com.tencent.supersonic.headless.api.pojo.DimValueMap;
-import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
-import com.tencent.supersonic.headless.api.pojo.RelateDimension;
-import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
-import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
+import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -21,11 +11,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 public class DataSetSchemaBuilder {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -29,10 +29,11 @@ public class ModelConverter {
     public static ModelDO convert(ModelReq modelReq, User user) {
         ModelDO modelDO = new ModelDO();
         // ModelDetail modelDetail = createModelDetail(modelReq);
+        ModelDetail modelDetail = modelReq.getModelDetail();
         modelReq.createdBy(user.getName());
         BeanMapper.mapper(modelReq, modelDO);
         modelDO.setStatus(StatusEnum.ONLINE.getCode());
-        modelDO.setModelDetail(JSONObject.toJSONString(modelReq.getModelDetail()));
+        modelDO.setModelDetail(JSONObject.toJSONString(modelDetail));
         modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions()));
         if (modelReq.getExt() != null) {
             modelDO.setExt(JSONObject.toJSONString(modelReq.getExt()));
@@ -123,9 +124,7 @@ public class ModelConverter {
         metricReq.setModelId(modelDO.getId());
         MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
         exprTypeParams.setExpr(measure.getExpr());
-        Measure measureParam = new Measure();
-        BeanMapper.mapper(measure, measureParam);
-        exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
+        exprTypeParams.setMeasures(Lists.newArrayList(measure));
         metricReq.setMetricDefineByMeasureParams(exprTypeParams);
         metricReq.setMetricDefineType(MetricDefineType.MEASURE);
         return metricReq;
@@ -165,11 +164,14 @@ public class ModelConverter {
                         getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
                 modelDetail.getIdentifiers().add(identify);
             } else if (FieldType.measure.equals(fieldType)) {
-                Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
-                        columnSchema.getAgg().getOperator(), 1);
+                Measure measure = new Measure(columnSchema.getName(),
+                        modelReq.getBizName() + "_" + columnSchema.getColumnName(),
+                        columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
+                Dimension dim = new Dimension(columnSchema.getName(),
+                        modelReq.getBizName() + "_" + columnSchema.getColumnName(),
+                        columnSchema.getColumnName(),
                         DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }
@@ -264,14 +266,17 @@ public class ModelConverter {
 
     private static ModelDetail createModelDetail(ModelReq modelReq) {
         ModelDetail modelDetail = new ModelDetail();
-        // List<Measure> measures = modelReq.getModelDetail().getMeasures();
-        // for (Measure measure : measures) {
-        // if (StringUtils.isBlank(measure.getBizName())) {
-        // continue;
-        // }
-        // measure.setExpr(measure.getBizName());
-        // measure.setBizName(String.format("%s_%", modelReq.getBizName(), measure.getExpr()));
-        // }
+        List<Measure> measures = modelReq.getModelDetail().getMeasures();
+        if (measures == null) {
+            measures = Lists.newArrayList();
+        }
+        for (Measure measure : measures) {
+            if (StringUtils.isBlank(measure.getBizName())) {
+                continue;
+            }
+            measure.setExpr(measure.getBizName());
+            measure.setBizName(String.format("%s_%s", modelReq.getBizName(), measure.getExpr()));
+        }
         BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
         return modelDetail;
     }