[improvement]Use QueryWrapper in place of hard-coded SQLs (#1944)

* [improvement][launcher]Use API to get element ID avoiding hard-code.

* [fix][launcher]Fix mysql scripts.

* [improvement][launcher]Support DuckDB database and refactor translator code structure.

* [improvement][headless-fe] Revamped the interaction for semantic modeling routing and successfully implemented the switching between dimension and dataset management.

* [improvement][Headless] Add table ddl in Dbschema

* [improvement][Headless] Add get database by type

* [improvement][Headless] Supports automatic batch creation of models based on db table names.

* [improvement][Headless] Supports getting domain by bizName

* [improvement][launcher]Refactor unit tests and demo data.

* [fix][launcher]Change default vector dimension to 512.

* [improvement](Dict) add dimValueAliasMap info for KnowledgeBaseService

* [improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml.

* [improvement][chat]Introduce ChatMemory to delegate ChatMemoryDO.

* [fix][common]Fix embedding store sys configs.

* [fix][common]Fix postgres schema, using varchar instead of char.

* [improvement][launcher]Change supersonic docker deployment from mysql to postgres.

* [Fix][launcher]Fix a number of issues related to semantic modeling.

* [Fix][headless]Fix the evaluation logic of agg type.

* [fix][assembly]Fix Dockerfile and add docker compose run script.

* [fix][chat]Fix "multiple assignments to same column "similar_queries".

* [improvement][headless]Use LamdaQueryWrapper to avoid hard-coded column names.

* [improvement][headless]Refactor headless infra to support advanced semantic modelling.

* [improvement][headless]Change class name `Dim` to `Dimension`.

* [improvement][chat]Introduce `TimeFieldMapper` to always map time field.

* [fix][headless]Remove unnecessary dimension existence check.

* [fix][chat]Fix adjusted filters don't take effect.

---------

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ColumnSchema.java

@@ -15,7 +15,7 @@ public class ColumnSchema {
 
     private FieldType filedType;
 
-    private AggOperatorEnum agg;
+    private AggOperatorEnum agg = AggOperatorEnum.SUM;
 
     private String name;
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java

@@ -9,7 +9,7 @@ import lombok.NoArgsConstructor;
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-public class Dim {
+public class Dimension {
 
     private String name;
 
@@ -27,16 +27,14 @@ public class Dim {
 
     private String description;
 
-    private int isTag;
-
-    public Dim(String name, String bizName, DimensionType type, Integer isCreateDimension) {
+    public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension) {
         this.name = name;
         this.type = type;
         this.isCreateDimension = isCreateDimension;
         this.bizName = bizName;
     }
 
-    public Dim(String name, String bizName, DimensionType type, Integer isCreateDimension,
+    public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension,
             String expr, String dateFormat, DimensionTimeTypeParams typeParams) {
         this.name = name;
         this.type = type;
@@ -47,8 +45,8 @@ public class Dim {
         this.bizName = bizName;
     }
 
-    public static Dim getDefault() {
-        return new Dim("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date",
+    public static Dimension getDefault() {
+        return new Dimension("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date",
                 Constants.DAY_FORMAT, new DimensionTimeTypeParams("false", "day"));
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -24,7 +24,7 @@ public class ModelDetail {
 
     private List<Identify> identifiers = Lists.newArrayList();
 
-    private List<Dim> dimensions = Lists.newArrayList();
+    private List<Dimension> dimensions = Lists.newArrayList();
 
     private List<Measure> measures = Lists.newArrayList();
 
@@ -39,7 +39,7 @@ public class ModelDetail {
         return sqlQuery;
     }
 
-    public List<Dim> filterTimeDims() {
+    public List<Dimension> filterTimeDims() {
         if (CollectionUtils.isEmpty(dimensions)) {
             return Lists.newArrayList();
         }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/FieldType.java

@@ -1,5 +1,5 @@
 package com.tencent.supersonic.headless.api.pojo.enums;
 
 public enum FieldType {
-    primary_key, foreign_key, partition_time, time, dimension, measure;
+    primary_key, foreign_key, partition_time, time, categorical, measure;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java

@@ -32,8 +32,6 @@ public class DimensionReq extends SchemaItem {
 
     private DataTypeEnums dataType;
 
-    private int isTag;
-
     private Map<String, Object> ext;
 
     private DimensionTimeTypeParams typeParams;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java

@@ -9,6 +9,10 @@ import java.util.List;
 @Data
 public class ModelBuildReq {
 
+    private String name;
+
+    private String bizName;
+
     private Long databaseId;
 
     private Long domainId;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -211,7 +211,9 @@ public class QueryStructReq extends SemanticQueryReq {
         SelectItem selectExpressionItem = new SelectItem(function);
         String alias =
                 StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
-        selectExpressionItem.setAlias(new Alias(alias));
+        if (!alias.equals(columnName)) {
+            selectExpressionItem.setAlias(new Alias(alias));
+        }
         return selectExpressionItem;
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -18,6 +18,8 @@ public class DimensionResp extends SchemaItem {
 
     private Long modelId;
 
+    private Long domainId;
+
     private DimensionType type;
 
     private String expr;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.headless.api.pojo.response;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.Field;
 import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -62,7 +62,7 @@ public class ModelResp extends SchemaItem {
         return isOpen != null && isOpen == 1;
     }
 
-    public List<Dim> getTimeDimension() {
+    public List<Dimension> getTimeDimension() {
         if (modelDetail == null) {
             return Lists.newArrayList();
         }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java

@@ -19,6 +19,7 @@ public class DictWord {
     private String word;
     private String nature;
     private String natureWithFrequency;
+    private String alias;
 
     @Override
     public boolean equals(Object o) {

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java

@@ -1,11 +1,14 @@
 package com.tencent.supersonic.headless.chat.knowledge;
 
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.headless.api.pojo.response.S2Term;
 import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -14,6 +17,31 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class KnowledgeBaseService {
+    private static volatile Map<Long, List<DictWord>> dimValueAliasMap = new HashMap<>();
+
+    public static Map<Long, List<DictWord>> getDimValueAlias() {
+        return dimValueAliasMap;
+    }
+
+    public static List<DictWord> addDimValueAlias(Long dimId, List<DictWord> newWords) {
+        List<DictWord> dimValueAlias =
+                dimValueAliasMap.containsKey(dimId) ? dimValueAliasMap.get(dimId)
+                        : new ArrayList<>();
+        Set<String> wordSet =
+                dimValueAlias
+                        .stream().map(word -> String.format("%s_%s_%s",
+                                word.getNatureWithFrequency(), word.getWord(), word.getAlias()))
+                        .collect(Collectors.toSet());
+        for (DictWord dictWord : newWords) {
+            String key = String.format("%s_%s_%s", dictWord.getNatureWithFrequency(),
+                    dictWord.getWord(), dictWord.getAlias());
+            if (!wordSet.contains(key)) {
+                dimValueAlias.add(dictWord);
+            }
+        }
+        dimValueAliasMap.put(dimId, dimValueAlias);
+        return dimValueAlias;
+    }
 
     public void updateSemanticKnowledge(List<DictWord> natures) {
 
@@ -41,6 +69,11 @@ public class KnowledgeBaseService {
         }
 
         // 2. update online knowledge
+        if (CollectionUtils.isNotEmpty(dimValueAliasMap)) {
+            for (Long dimId : dimValueAliasMap.keySet()) {
+                natures.addAll(dimValueAliasMap.get(dimId));
+            }
+        }
         updateOnlineKnowledge(natures);
     }
 

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java

@@ -12,6 +12,8 @@ import com.hankcs.hanlp.dictionary.other.CharTable;
 import com.hankcs.hanlp.seg.common.Term;
 import com.hankcs.hanlp.utility.LexiconUtility;
 import com.hankcs.hanlp.utility.TextUtility;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
 
 import java.io.BufferedOutputStream;
@@ -103,7 +105,22 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
                 String word = getWordBySpace(param[0]);
                 if (isLetters) {
                     original = word;
-                    word = word.toLowerCase();
+                    // word = word.toLowerCase();
+                    // 加入小写别名
+                    if (!original.equals(word.toLowerCase())) {
+                        DictWord dictWord = new DictWord();
+                        String nature = param[1];
+                        dictWord.setNatureWithFrequency(
+                                String.format("%s " + Constants.DEFAULT_FREQUENCY, nature));
+                        dictWord.setWord(word);
+                        dictWord.setAlias(word.toLowerCase());
+                        String[] split = nature.split(DictWordType.NATURE_SPILT);
+                        if (split.length >= 2) {
+                            Long dimId = Long.parseLong(
+                                    nature.split(DictWordType.NATURE_SPILT)[split.length - 1]);
+                            KnowledgeBaseService.addDimValueAlias(dimId, Arrays.asList(dictWord));
+                        }
+                    }
                 }
                 if (natureCount == 0) {
                     attribute = new CoreDictionary.Attribute(defaultNature);

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java

@@ -8,12 +8,15 @@ import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.headless.api.pojo.response.S2Term;
 import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import com.tencent.supersonic.headless.chat.knowledge.DatabaseMapResult;
+import com.tencent.supersonic.headless.chat.knowledge.DictWord;
 import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
 import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
 import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.chat.utils.EditDistanceUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
 import java.util.HashSet;
@@ -83,12 +86,32 @@ public class KeywordMapper extends BaseMapper {
                         .element(element).frequency(frequency).word(hanlpMapResult.getName())
                         .similarity(hanlpMapResult.getSimilarity())
                         .detectWord(hanlpMapResult.getDetectWord()).build();
-
+                // doDimValueAliasLogic 将维度值别名进行替换成真实维度值
+                doDimValueAliasLogic(schemaElementMatch);
                 addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch);
             }
         }
     }
 
+    private void doDimValueAliasLogic(SchemaElementMatch schemaElementMatch) {
+        SchemaElement element = schemaElementMatch.getElement();
+        if (SchemaElementType.VALUE.equals(element.getType())) {
+            Long dimId = element.getId();
+            String word = schemaElementMatch.getWord();
+            Map<Long, List<DictWord>> dimValueAlias = KnowledgeBaseService.getDimValueAlias();
+            if (Objects.nonNull(dimId) && StringUtils.isNotEmpty(word)
+                    && dimValueAlias.containsKey(dimId)) {
+                Map<String, DictWord> aliasAndDictMap = dimValueAlias.get(dimId).stream()
+                        .collect(Collectors.toMap(dictWord -> dictWord.getAlias(),
+                                dictWord -> dictWord, (v1, v2) -> v2));
+                if (aliasAndDictMap.containsKey(word)) {
+                    String wordTech = aliasAndDictMap.get(word).getWord();
+                    schemaElementMatch.setWord(wordTech);
+                }
+            }
+        }
+    }
+
     private void convertMapResultToMapInfo(ChatQueryContext chatQueryContext,
             List<DatabaseMapResult> mapResults) {
         for (DatabaseMapResult match : mapResults) {

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/TimeFieldMapper.java

@@ -0,0 +1,37 @@
+package com.tencent.supersonic.headless.chat.mapper;
+
+import com.tencent.supersonic.common.pojo.enums.Text2SQLType;
+import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
+import com.tencent.supersonic.headless.chat.ChatQueryContext;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class TimeFieldMapper extends BaseMapper {
+
+    @Override
+    public void doMap(ChatQueryContext chatQueryContext) {
+        if (chatQueryContext.getRequest().getText2SQLType().equals(Text2SQLType.ONLY_RULE)) {
+            return;
+        }
+
+        Map<Long, DataSetSchema> schemaMap =
+                chatQueryContext.getSemanticSchema().getDataSetSchemaMap();
+        for (Map.Entry<Long, DataSetSchema> entry : schemaMap.entrySet()) {
+            List<SchemaElement> timeDims = entry.getValue().getDimensions().stream()
+                    .filter(dim -> dim.getTimeFormat() != null).collect(Collectors.toList());
+            for (SchemaElement schemaElement : timeDims) {
+                chatQueryContext.getMapInfo().getMatchedElements(entry.getKey())
+                        .add(SchemaElementMatch.builder().word(schemaElement.getName())
+                                .element(schemaElement).detectWord(schemaElement.getName())
+                                .similarity(1.0).build());
+            }
+        }
+    }
+
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java

@@ -84,7 +84,7 @@ public abstract class BaseDbAdaptor implements DbAdaptor {
         return connection.getMetaData();
     }
 
-    protected static FieldType classifyColumnType(String typeName) {
+    public FieldType classifyColumnType(String typeName) {
         switch (typeName.toUpperCase()) {
             case "INT":
             case "INTEGER":
@@ -101,7 +101,7 @@ public abstract class BaseDbAdaptor implements DbAdaptor {
             case "TIMESTAMP":
                 return FieldType.time;
             default:
-                return FieldType.dimension;
+                return FieldType.categorical;
         }
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.core.adaptor.db;
 
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
+import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
 import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
 
 import java.sql.SQLException;
@@ -19,4 +20,6 @@ public interface DbAdaptor {
 
     List<DBColumn> getColumns(ConnectInfo connectInfo, String schemaName, String tableName)
             throws SQLException;
+
+    FieldType classifyColumnType(String typeName);
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java

@@ -114,7 +114,8 @@ public class PostgresqlAdaptor extends BaseDbAdaptor {
         return dbColumns;
     }
 
-    protected static FieldType classifyColumnType(String typeName) {
+    @Override
+    public FieldType classifyColumnType(String typeName) {
         switch (typeName.toUpperCase()) {
             case "INT":
             case "INTEGER":
@@ -141,7 +142,7 @@ public class PostgresqlAdaptor extends BaseDbAdaptor {
             case "CHARACTER":
             case "UUID":
             default:
-                return FieldType.dimension;
+                return FieldType.categorical;
         }
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java

@@ -61,13 +61,18 @@ public class SqlQueryConverter implements QueryConverter {
         List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
         List<String> metrics =
                 metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList());
-        AggOption aggOption = getAggOption(sqlQueryParam.getSql(), metricSchemas);
         Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
         OntologyQueryParam ontologyQueryParam = new OntologyQueryParam();
         ontologyQueryParam.getMetrics().addAll(metrics);
         ontologyQueryParam.getDimensions().addAll(dimensions);
-        ontologyQueryParam.setAggOption(aggOption);
-        ontologyQueryParam.setNativeQuery(!AggOption.isAgg(aggOption));
+        AggOption sqlQueryAggOption = getAggOption(sqlQueryParam.getSql(), metricSchemas);
+        // if sql query itself has aggregation, ontology query just returns detail
+        if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) {
+            ontologyQueryParam.setAggOption(AggOption.NATIVE);
+        } else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !metrics.isEmpty()) {
+            ontologyQueryParam.setAggOption(AggOption.DEFAULT);
+        }
+        ontologyQueryParam.setNativeQuery(!AggOption.isAgg(ontologyQueryParam.getAggOption()));
         queryStatement.setOntologyQueryParam(ontologyQueryParam);
 
         generateDerivedMetric(sqlGenerateUtils, queryStatement);

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java

@@ -57,7 +57,11 @@ public class StructQueryConverter implements QueryConverter {
                 .map(Aggregator::getColumn).collect(Collectors.toList()));
         String where = sqlGenerateUtils.generateWhere(structQueryParam, null);
         ontologyQueryParam.setWhere(where);
-        ontologyQueryParam.setAggOption(AggOption.AGGREGATION);
+        if (ontologyQueryParam.getMetrics().isEmpty()) {
+            ontologyQueryParam.setAggOption(AggOption.NATIVE);
+        } else {
+            ontologyQueryParam.setAggOption(AggOption.DEFAULT);
+        }
         ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery());
         ontologyQueryParam.setOrder(structQueryParam.getOrders().stream()
                 .map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java

@@ -26,8 +26,8 @@ public class DataModelNode extends SemanticNode {
             sqlTable = dataModel.getSqlQuery();
         } else if (dataModel.getTableQuery() != null && !dataModel.getTableQuery().isEmpty()) {
             if (dataModel.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
-                String fullTableName = Arrays.stream(dataModel.getTableQuery().split("\\."))
-                        .collect(Collectors.joining(".public."));
+                String fullTableName =
+                        String.join(".public.", dataModel.getTableQuery().split("\\."));
                 sqlTable = "select * from " + fullTableName;
             } else {
                 sqlTable = "select * from " + dataModel.getTableQuery();
@@ -64,7 +64,7 @@ public class DataModelNode extends SemanticNode {
         for (Dimension d : datasource.getDimensions()) {
             List<SqlNode> identifiers =
                     expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope);
-            identifiers.stream().forEach(i -> dimensions.add(i.toString()));
+            identifiers.forEach(i -> dimensions.add(i.toString()));
             dimensions.add(d.getName());
         }
         for (Identify i : datasource.getIdentifiers()) {
@@ -73,7 +73,7 @@ public class DataModelNode extends SemanticNode {
         for (Measure m : datasource.getMeasures()) {
             List<SqlNode> identifiers =
                     expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope);
-            identifiers.stream().forEach(i -> {
+            identifiers.forEach(i -> {
                 if (!dimensions.contains(i.toString())) {
                     metrics.add(i.toString());
                 }
@@ -127,7 +127,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static String getNames(List<DataModel> dataModelList) {
-        return dataModelList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
+        return dataModelList.stream().map(DataModel::getName).collect(Collectors.joining("_"));
     }
 
     public static void getQueryDimensionMeasure(Ontology ontology, OntologyQueryParam queryParam,
@@ -138,12 +138,12 @@ public class DataModelNode extends SemanticNode {
                         : d)
                 .collect(Collectors.toSet()));
         Set<String> schemaMetricName =
-                ontology.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
+                ontology.getMetrics().stream().map(Metric::getName).collect(Collectors.toSet());
         ontology.getMetrics().stream().filter(m -> queryParam.getMetrics().contains(m.getName()))
-                .forEach(m -> m.getMetricTypeParams().getMeasures().stream()
+                .forEach(m -> m.getMetricTypeParams().getMeasures()
                         .forEach(mm -> queryMeasures.add(mm.getName())));
         queryParam.getMetrics().stream().filter(m -> !schemaMetricName.contains(m))
-                .forEach(m -> queryMeasures.add(m));
+                .forEach(queryMeasures::add);
     }
 
     public static void mergeQueryFilterDimensionMeasure(Ontology ontology,
@@ -155,13 +155,13 @@ public class DataModelNode extends SemanticNode {
             FilterNode.getFilterField(parse(queryParam.getWhere(), scope, engineType),
                     filterConditions);
             Set<String> queryMeasures = new HashSet<>(measures);
-            Set<String> schemaMetricName = ontology.getMetrics().stream().map(m -> m.getName())
-                    .collect(Collectors.toSet());
+            Set<String> schemaMetricName =
+                    ontology.getMetrics().stream().map(Metric::getName).collect(Collectors.toSet());
             for (String filterCondition : filterConditions) {
                 if (schemaMetricName.contains(filterCondition)) {
                     ontology.getMetrics().stream()
                             .filter(m -> m.getName().equalsIgnoreCase(filterCondition))
-                            .forEach(m -> m.getMetricTypeParams().getMeasures().stream()
+                            .forEach(m -> m.getMetricTypeParams().getMeasures()
                                     .forEach(mm -> queryMeasures.add(mm.getName())));
                     continue;
                 }
@@ -196,8 +196,8 @@ public class DataModelNode extends SemanticNode {
         }
 
         // second, traverse the ontology to find other related dataModels
-        List<DataModel> relatedDataModels = findRelatedModelsByRelation(ontology, baseDataModel,
-                queryDimensions, queryMeasures);
+        List<DataModel> relatedDataModels = findRelatedModelsByRelation(ontology, queryParam,
+                baseDataModel, queryDimensions, queryMeasures);
         if (CollectionUtils.isEmpty(relatedDataModels)) {
             relatedDataModels = findRelatedModelsByIdentifier(ontology, baseDataModel,
                     queryDimensions, queryMeasures);
@@ -255,7 +255,7 @@ public class DataModelNode extends SemanticNode {
                 .collect(Collectors.toSet());
         Set<String> baseDimensions = baseDataModel.getDimensions().stream().map(Dimension::getName)
                 .collect(Collectors.toSet());
-        baseDataModel.getIdentifiers().stream().forEach(i -> baseDimensions.add(i.getName()));
+        baseDataModel.getIdentifiers().forEach(i -> baseDimensions.add(i.getName()));
 
         baseMeasures.retainAll(queryMeasures);
         if (baseMeasures.size() < queryMeasures.size()) {
@@ -282,7 +282,8 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static List<DataModel> findRelatedModelsByRelation(Ontology ontology,
-            DataModel baseDataModel, Set<String> queryDimensions, Set<String> queryMeasures) {
+            OntologyQueryParam queryParam, DataModel baseDataModel, Set<String> queryDimensions,
+            Set<String> queryMeasures) {
         Set<String> joinDataModelNames = new HashSet<>();
         List<DataModel> joinDataModels = new ArrayList<>();
         Set<String> before = new HashSet<>();
@@ -295,7 +296,7 @@ public class DataModelNode extends SemanticNode {
                     visitJoinRelations, sortedJoinRelation);
             ontology.getJoinRelations().stream()
                     .filter(j -> !visitJoinRelations.contains(j.getId()))
-                    .forEach(j -> sortedJoinRelation.add(j));
+                    .forEach(sortedJoinRelation::add);
             for (JoinRelation joinRelation : sortedJoinRelation) {
                 if (!before.contains(joinRelation.getLeft())
                         && !before.contains(joinRelation.getRight())) {
@@ -305,13 +306,17 @@ public class DataModelNode extends SemanticNode {
                 boolean isRight = before.contains(joinRelation.getLeft());
                 DataModel other = isRight ? ontology.getDataModelMap().get(joinRelation.getRight())
                         : ontology.getDataModelMap().get(joinRelation.getLeft());
+                String joinDimName = isRight ? joinRelation.getJoinCondition().get(0).getRight()
+                        : joinRelation.getJoinCondition().get(0).getLeft();
                 if (!queryDimensions.isEmpty()) {
                     Set<String> linkDimension = other.getDimensions().stream()
-                            .map(dd -> dd.getName()).collect(Collectors.toSet());
-                    other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName()));
+                            .map(Dimension::getName).collect(Collectors.toSet());
+                    other.getIdentifiers().forEach(i -> linkDimension.add(i.getName()));
                     linkDimension.retainAll(queryDimensions);
                     if (!linkDimension.isEmpty()) {
                         isMatch = true;
+                        // joinDim should be added to the query dimension
+                        queryParam.getDimensions().add(joinDimName);
                     }
                 }
                 Set<String> linkMeasure = other.getMeasures().stream().map(Measure::getName)
@@ -322,7 +327,7 @@ public class DataModelNode extends SemanticNode {
                 }
                 if (!isMatch && ontology.getDimensionMap().containsKey(other.getName())) {
                     Set<String> linkDimension = ontology.getDimensionMap().get(other.getName())
-                            .stream().map(dd -> dd.getName()).collect(Collectors.toSet());
+                            .stream().map(Dimension::getName).collect(Collectors.toSet());
                     linkDimension.retainAll(queryDimensions);
                     if (!linkDimension.isEmpty()) {
                         isMatch = true;
@@ -382,15 +387,14 @@ public class DataModelNode extends SemanticNode {
             if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) {
                 continue;
             }
-            Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName())
-                    .filter(i -> baseIdentifiers.contains(i)).count();
+            long identifierNum = entry.getValue().getIdentifiers().stream().map(Identify::getName)
+                    .filter(baseIdentifiers::contains).count();
             if (identifierNum > 0) {
                 boolean isMatch = false;
                 if (!queryDimension.isEmpty()) {
                     Set<String> linkDimension = entry.getValue().getDimensions().stream()
-                            .map(dd -> dd.getName()).collect(Collectors.toSet());
-                    entry.getValue().getIdentifiers().stream()
-                            .forEach(i -> linkDimension.add(i.getName()));
+                            .map(Dimension::getName).collect(Collectors.toSet());
+                    entry.getValue().getIdentifiers().forEach(i -> linkDimension.add(i.getName()));
                     linkDimension.retainAll(queryDimension);
                     if (!linkDimension.isEmpty()) {
                         isMatch = true;
@@ -398,7 +402,7 @@ public class DataModelNode extends SemanticNode {
                 }
                 if (!measures.isEmpty()) {
                     Set<String> linkMeasure = entry.getValue().getMeasures().stream()
-                            .map(mm -> mm.getName()).collect(Collectors.toSet());
+                            .map(Measure::getName).collect(Collectors.toSet());
                     linkMeasure.retainAll(measures);
                     if (!linkMeasure.isEmpty()) {
                         isMatch = true;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java

@@ -15,6 +15,6 @@ public class OntologyQueryParam {
     private String where;
     private Long limit;
     private List<ColumnOrder> order;
-    private boolean nativeQuery = false;
-    private AggOption aggOption = AggOption.DEFAULT;
+    private boolean nativeQuery = true;
+    private AggOption aggOption = AggOption.NATIVE;
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.headless.core.utils;
 
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
@@ -20,11 +20,11 @@ public class SysTimeDimensionBuilder {
             Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
                     Pattern.CASE_INSENSITIVE);
 
-    public static void addSysTimeDimension(List<Dim> dims, DbAdaptor engineAdaptor) {
+    public static void addSysTimeDimension(List<Dimension> dims, DbAdaptor engineAdaptor) {
         log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
-        Dim timeDim = getTimeDim(dims);
+        Dimension timeDim = getTimeDim(dims);
         if (timeDim == null) {
-            timeDim = Dim.getDefault();
+            timeDim = Dimension.getDefault();
             // todo not find the time dimension
             return;
         }
@@ -34,8 +34,8 @@ public class SysTimeDimensionBuilder {
         log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor);
     }
 
-    private static Dim generateSysDayDimension(Dim timeDim, DbAdaptor engineAdaptor) {
-        Dim dim = new Dim();
+    private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
+        Dimension dim = new Dimension();
         dim.setBizName(TimeDimensionEnum.DAY.getName());
         dim.setType(DimensionType.partition_time);
         dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(),
@@ -47,8 +47,8 @@ public class SysTimeDimensionBuilder {
         return dim;
     }
 
-    private static Dim generateSysWeekDimension(Dim timeDim, DbAdaptor engineAdaptor) {
-        Dim dim = new Dim();
+    private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
+        Dimension dim = new Dimension();
         dim.setBizName(TimeDimensionEnum.WEEK.getName());
         dim.setType(DimensionType.partition_time);
         dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(),
@@ -60,8 +60,8 @@ public class SysTimeDimensionBuilder {
         return dim;
     }
 
-    private static Dim generateSysMonthDimension(Dim timeDim, DbAdaptor engineAdaptor) {
-        Dim dim = new Dim();
+    private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
+        Dimension dim = new Dimension();
         dim.setBizName(TimeDimensionEnum.MONTH.getName());
         dim.setType(DimensionType.partition_time);
         dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(),
@@ -79,7 +79,8 @@ public class SysTimeDimensionBuilder {
     }
 
     // Check whether the time field contains keywords,Generation time expression
-    private static String generateTimeExpr(Dim timeDim, String dateType, DbAdaptor engineAdaptor) {
+    private static String generateTimeExpr(Dimension timeDim, String dateType,
+            DbAdaptor engineAdaptor) {
         String bizName = timeDim.getBizName();
         String dateFormat = timeDim.getDateFormat();
         if (containsTimeKeyword(bizName)) {
@@ -90,8 +91,8 @@ public class SysTimeDimensionBuilder {
         }
     }
 
-    private static Dim getTimeDim(List<Dim> timeDims) {
-        for (Dim dim : timeDims) {
+    private static Dimension getTimeDim(List<Dimension> timeDims) {
+        for (Dimension dim : timeDims) {
             if (dim.getType().equals(DimensionType.partition_time)) {
                 return dim;
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -207,7 +207,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
         ModelResp modelResp = modelResps.get(0);
         String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
                 modelResp.getName());
-        List<Dim> timeDims = modelResp.getTimeDimension();
+        List<Dimension> timeDims = modelResp.getTimeDimension();
         if (CollectionUtils.isNotEmpty(timeDims)) {
             sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
                     TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),

headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java

@@ -27,7 +27,7 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
             dictWord.setWord(dataItem.getName());
             String sign = DictWordType.NATURE_SPILT;
             String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
-            String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
+            String nature = sign + dataItem.getModelId() + sign + dataItem.getId() + suffixNature;
             String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
             dictWord.setNature(nature);
             dictWord.setNatureWithFrequency(natureWithFrequency);

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.headless.server.manager;
 
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
@@ -18,7 +18,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -54,7 +53,7 @@ public class ModelYamlManager {
         return dataModelYamlTpl;
     }
 
-    public static DimensionYamlTpl convert(Dim dim) {
+    public static DimensionYamlTpl convert(Dimension dim) {
         DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
         BeanUtils.copyProperties(dim, dimensionYamlTpl);
         dimensionYamlTpl.setName(dim.getBizName());
@@ -85,15 +84,4 @@ public class ModelYamlManager {
         return identifyYamlTpl;
     }
 
-    private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) {
-        Measure measure = new Measure();
-        measure.setExpr("1");
-        if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
-            measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
-        }
-        measure.setAgg("count");
-        measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
-        measure.setIsCreateMetric(1);
-        datasourceDetail.getMeasures().add(measure);
-    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -23,7 +23,7 @@ public class RuleSemanticModeller implements SemanticModeller {
 
     private ColumnSchema convert(DBColumn dbColumn) {
         ColumnSchema columnSchema = new ColumnSchema();
-        columnSchema.setName(dbColumn.getComment());
+        columnSchema.setName(dbColumn.getColumnName());
         columnSchema.setColumnName(dbColumn.getColumnName());
         columnSchema.setComment(dbColumn.getComment());
         columnSchema.setDataType(dbColumn.getDataType());

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java

@@ -16,6 +16,7 @@ public class ModelDO {
 
     private Long domainId;
 
+    @Deprecated
     private Long tagObjectId;
 
     private String name;

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java

@@ -28,6 +28,7 @@ public class QueryStatDO {
     private String queryStructCmd;
     @TableField("struct_cmd_md5")
     private String queryStructCmdMd5;
+    @TableField("\"sql\"")
     private String sql;
     private String sqlMd5;
     private String queryEngine;

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
-import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import org.apache.ibatis.annotations.Mapper;
 
@@ -12,11 +11,7 @@ public interface DimensionDOCustomMapper {
 
     void batchInsert(List<DimensionDO> dimensionDOS);
 
-    void batchUpdate(List<DimensionDO> dimensionDOS);
-
     void batchUpdateStatus(List<DimensionDO> dimensionDOS);
 
-    List<DimensionDO> query(DimensionFilter dimensionFilter);
-
     List<DimensionDO> queryDimensions(DimensionsFilter dimensionsFilter);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
-import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
 import org.apache.ibatis.annotations.Mapper;
 
@@ -20,7 +19,5 @@ public interface MetricDOCustomMapper {
 
     void updateClassificationsBatch(List<MetricDO> metricDOS);
 
-    List<MetricDO> query(MetricFilter metricFilter);
-
     List<MetricDO> queryMetrics(MetricsFilter metricsFilter);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java

@@ -1,15 +1,9 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.tencent.supersonic.headless.api.pojo.QueryStat;
-import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.server.persistence.dataobject.QueryStatDO;
 import org.apache.ibatis.annotations.Mapper;
 
-import java.util.List;
-
 @Mapper
 public interface StatMapper extends BaseMapper<QueryStatDO> {
-
-    List<QueryStat> getStatInfo(ItemUseReq itemUseCommend);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -118,6 +118,7 @@ public class DictRepositoryImpl implements DictRepository {
             wrapper.lambda().and(qw -> qw.like(DictTaskDO::getName, key).or()
                     .like(DictTaskDO::getDescription, key).or().like(DictTaskDO::getConfig, key));
         }
+        wrapper.lambda().orderByDesc(DictTaskDO::getCreatedAt);
         return dictTaskMapper.selectList(wrapper);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java

@@ -1,14 +1,17 @@
 package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCustomMapper;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Objects;
 
 @Service
 public class DimensionRepositoryImpl implements DimensionRepository {
@@ -50,7 +53,43 @@ public class DimensionRepositoryImpl implements DimensionRepository {
 
     @Override
     public List<DimensionDO> getDimension(DimensionFilter dimensionFilter) {
-        return dimensionDOCustomMapper.query(dimensionFilter);
+        QueryWrapper<DimensionDO> queryWrapper = new QueryWrapper<>();
+        queryWrapper.lambda().ne(DimensionDO::getStatus, 3);
+        if (Objects.nonNull(dimensionFilter.getIds()) && !dimensionFilter.getIds().isEmpty()) {
+            queryWrapper.lambda().in(DimensionDO::getId, dimensionFilter.getIds());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getId())) {
+            queryWrapper.lambda().eq(DimensionDO::getId, dimensionFilter.getId());
+        }
+        if (Objects.nonNull(dimensionFilter.getModelIds())
+                && !dimensionFilter.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(DimensionDO::getModelId, dimensionFilter.getModelIds());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getName())) {
+            queryWrapper.lambda().like(DimensionDO::getName, dimensionFilter.getName());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getId())) {
+            queryWrapper.lambda().like(DimensionDO::getBizName, dimensionFilter.getBizName());
+        }
+        if (Objects.nonNull(dimensionFilter.getStatus())) {
+            queryWrapper.lambda().eq(DimensionDO::getStatus, dimensionFilter.getStatus());
+        }
+        if (Objects.nonNull(dimensionFilter.getSensitiveLevel())) {
+            queryWrapper.lambda().eq(DimensionDO::getSensitiveLevel,
+                    dimensionFilter.getSensitiveLevel());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getCreatedBy())) {
+            queryWrapper.lambda().eq(DimensionDO::getCreatedBy, dimensionFilter.getCreatedBy());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getKey())) {
+            String key = dimensionFilter.getKey();
+            queryWrapper.lambda().like(DimensionDO::getName, key).or()
+                    .like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key)
+                    .or().like(DimensionDO::getAlias, key).or()
+                    .like(DimensionDO::getCreatedBy, key);
+        }
+
+        return dimensionDOMapper.selectList(queryWrapper);
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java

@@ -9,9 +9,11 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefa
 import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
 import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.Objects;
 
 @Component
 public class MetricRepositoryImpl implements MetricRepository {
@@ -73,7 +75,46 @@ public class MetricRepositoryImpl implements MetricRepository {
 
     @Override
     public List<MetricDO> getMetric(MetricFilter metricFilter) {
-        return metricDOCustomMapper.query(metricFilter);
+        QueryWrapper<MetricDO> queryWrapper = new QueryWrapper<>();
+        queryWrapper.lambda().ne(MetricDO::getStatus, 3);
+        if (Objects.nonNull(metricFilter.getIds()) && !metricFilter.getIds().isEmpty()) {
+            queryWrapper.lambda().in(MetricDO::getId, metricFilter.getIds());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getId())) {
+            queryWrapper.lambda().eq(MetricDO::getId, metricFilter.getId());
+        }
+        if (Objects.nonNull(metricFilter.getModelIds()) && !metricFilter.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(MetricDO::getModelId, metricFilter.getModelIds());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getType())) {
+            queryWrapper.lambda().eq(MetricDO::getType, metricFilter.getType());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getName())) {
+            queryWrapper.lambda().like(MetricDO::getName, metricFilter.getName());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getId())) {
+            queryWrapper.lambda().like(MetricDO::getBizName, metricFilter.getBizName());
+        }
+        if (Objects.nonNull(metricFilter.getStatus())) {
+            queryWrapper.lambda().eq(MetricDO::getStatus, metricFilter.getStatus());
+        }
+        if (Objects.nonNull(metricFilter.getSensitiveLevel())) {
+            queryWrapper.lambda().eq(MetricDO::getSensitiveLevel, metricFilter.getSensitiveLevel());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) {
+            queryWrapper.lambda().eq(MetricDO::getCreatedBy, metricFilter.getCreatedBy());
+        }
+        if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) {
+            queryWrapper.lambda().eq(MetricDO::getIsPublish, metricFilter.getIsPublish());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getKey())) {
+            String key = metricFilter.getKey();
+            queryWrapper.lambda().like(MetricDO::getName, key).or().like(MetricDO::getBizName, key)
+                    .or().like(MetricDO::getDescription, key).or().like(MetricDO::getAlias, key)
+                    .or().like(MetricDO::getCreatedBy, key);
+        }
+
+        return metricDOMapper.selectList(queryWrapper);
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java

@@ -51,7 +51,7 @@ public class QueryRuleRepositoryImpl implements QueryRuleRepository {
 
         QueryWrapper<QueryRuleDO> wrapperSys = new QueryWrapper<>();
         // 返回系统设置的规则
-        wrapperSys.or().eq("priority", 0L);
+        wrapperSys.lambda().or().eq(QueryRuleDO::getPriority, 0L);
         List<QueryRuleDO> queryRuleDOListSys = mapper.selectList(wrapperSys);
 
         queryRuleDOList.addAll(queryRuleDOListSys);

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -47,7 +48,7 @@ public class StatRepositoryImpl implements StatRepository {
     @SneakyThrows
     public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
         List<ItemUseResp> result = new ArrayList<>();
-        List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
+        List<QueryStatDO> statInfos = getQueryStats(itemUseReq);
         Map<String, Long> map = new ConcurrentHashMap<>();
         statInfos.stream().forEach(stat -> {
             String dimensions = stat.getDimensions();
@@ -70,6 +71,21 @@ public class StatRepositoryImpl implements StatRepository {
                 .collect(Collectors.toList());
     }
 
+    private List<QueryStatDO> getQueryStats(ItemUseReq itemUseReq) {
+        QueryWrapper<QueryStatDO> queryWrapper = new QueryWrapper<>();
+        if (Objects.nonNull(itemUseReq.getModelId())) {
+            queryWrapper.lambda().eq(QueryStatDO::getModelId, itemUseReq.getModelId());
+        }
+        if (Objects.nonNull(itemUseReq.getModelIds()) && !itemUseReq.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(QueryStatDO::getModelId, itemUseReq.getModelIds());
+        }
+        if (Objects.nonNull(itemUseReq.getMetric())) {
+            queryWrapper.lambda().like(QueryStatDO::getMetrics, itemUseReq.getMetric());
+        }
+
+        return statMapper.selectList(queryWrapper);
+    }
+
     private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type,
             Long dataSetId) {
         if (StringUtils.isNotEmpty(dimensions)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java

@@ -47,7 +47,7 @@ public interface DimensionService {
 
     void sendDimensionEventBatch(List<Long> modelIds, EventType eventType);
 
-    DataEvent getDataEvent();
+    DataEvent getAllDataEvents();
 
     Boolean updateDimValueAlias(DimValueAliasReq req, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java

@@ -79,7 +79,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
         DataSetDO dataSetDO = convert(dataSetReq);
         dataSetDO.setStatus(StatusEnum.ONLINE.getCode());
         DataSetResp dataSetResp = convert(dataSetDO);
-        conflictCheck(dataSetResp);
+        // conflictCheck(dataSetResp);
         save(dataSetDO);
         dataSetResp.setId(dataSetDO.getId());
         return dataSetResp;
@@ -90,7 +90,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
         dataSetReq.updatedBy(user.getName());
         DataSetDO dataSetDO = convert(dataSetReq);
         DataSetResp dataSetResp = convert(dataSetDO);
-        conflictCheck(dataSetResp);
+        // conflictCheck(dataSetResp);
         updateById(dataSetDO);
         return dataSetResp;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -225,6 +225,9 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
         if (StringUtils.isNotBlank(modelBuildReq.getSql())) {
             List<DBColumn> columns =
                     getColumns(modelBuildReq.getDatabaseId(), modelBuildReq.getSql());
+            DatabaseResp databaseResp = getDatabase(modelBuildReq.getDatabaseId());
+            DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
+            columns.forEach(c -> c.setFieldType(engineAdaptor.classifyColumnType(c.getDataType())));
             dbColumnMap.put(modelBuildReq.getSql(), columns);
         } else {
             for (String table : modelBuildReq.getTables()) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -6,12 +6,9 @@ import com.alibaba.fastjson.TypeReference;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DataEvent;
-import com.tencent.supersonic.common.pojo.DataItem;
-import com.tencent.supersonic.common.pojo.ModelRela;
-import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -25,23 +22,14 @@ import com.tencent.supersonic.headless.api.pojo.request.DimValueAliasReq;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
-import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import com.tencent.supersonic.headless.server.pojo.ModelFilter;
-import com.tencent.supersonic.headless.server.service.DataSetService;
-import com.tencent.supersonic.headless.server.service.DatabaseService;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.ModelRelaService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.service.*;
 import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
 import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
@@ -152,11 +140,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         DimensionConverter.convert(dimensionDO, dimensionReq);
         dimensionRepository.updateDimension(dimensionDO);
         if (!oldName.equals(dimensionDO.getName())) {
-            sendEvent(
-                    DataItem.builder().modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
-                            .newName(dimensionReq.getName()).name(oldName).type(TypeEnums.DIMENSION)
-                            .id(dimensionDO.getId() + Constants.UNDERLINE).build(),
-                    EventType.UPDATE);
+            sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
         }
     }
 
@@ -424,7 +408,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         eventPublisher.publishEvent(dataEvent);
     }
 
-    public DataEvent getDataEvent() {
+    public DataEvent getAllDataEvents() {
         DimensionFilter dimensionFilter = new DimensionFilter();
         List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
         return getDataEvent(dimensionDOS, EventType.ADD);
@@ -464,13 +448,18 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         return true;
     }
 
+    private DataItem getDataItem(DimensionDO dimensionDO) {
+        ModelResp modelResp = modelService.getModel(dimensionDO.getModelId());
+        DimensionResp dimensionResp = DimensionConverter.convert2DimensionResp(dimensionDO,
+                ImmutableMap.of(modelResp.getId(), modelResp));
+        return DataItem.builder().id(dimensionResp.getId().toString()).name(dimensionResp.getName())
+                .bizName(dimensionResp.getBizName()).modelId(dimensionResp.getModelId().toString())
+                .domainId(dimensionResp.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
+    }
+
     private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
-        List<DataItem> dataItems = dimensionDOS.stream()
-                .map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE)
-                        .name(dimensionDO.getName())
-                        .modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
-                        .type(TypeEnums.DIMENSION).build())
-                .collect(Collectors.toList());
+        List<DataItem> dataItems =
+                dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList());
         return new DataEvent(this, dataItems, eventType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -5,59 +5,26 @@ import com.alibaba.fastjson.TypeReference;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DataEvent;
-import com.tencent.supersonic.common.pojo.DataItem;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.*;
 import com.tencent.supersonic.common.util.BeanMapper;
-import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
-import com.tencent.supersonic.headless.api.pojo.Measure;
-import com.tencent.supersonic.headless.api.pojo.MeasureParam;
-import com.tencent.supersonic.headless.api.pojo.MetaFilter;
-import com.tencent.supersonic.headless.api.pojo.MetricParam;
-import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
-import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
-import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
-import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.request.*;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
-import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
-import com.tencent.supersonic.headless.server.pojo.MetricFilter;
-import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
-import com.tencent.supersonic.headless.server.pojo.ModelCluster;
-import com.tencent.supersonic.headless.server.pojo.ModelFilter;
-import com.tencent.supersonic.headless.server.service.CollectService;
-import com.tencent.supersonic.headless.server.service.DataSetService;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.pojo.*;
+import com.tencent.supersonic.headless.server.service.*;
 import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
 import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -70,18 +37,7 @@ import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 @Service
@@ -667,12 +623,13 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
     }
 
     private DataItem getDataItem(MetricDO metricDO) {
-        MetricResp metricResp =
-                MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
+        ModelResp modelResp = modelService.getModel(metricDO.getModelId());
+        MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO,
+                ImmutableMap.of(modelResp.getId(), modelResp), Lists.newArrayList());
         fillDefaultAgg(metricResp);
-        return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE)
-                .name(metricDO.getName()).bizName(metricDO.getBizName())
-                .modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC)
+        return DataItem.builder().id(metricResp.getId().toString()).name(metricResp.getName())
+                .bizName(metricResp.getBizName()).modelId(metricResp.getModelId().toString())
+                .domainId(metricResp.getDomainId().toString()).type(TypeEnums.METRIC)
                 .defaultAgg(metricResp.getDefaultAgg()).build();
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.DbSchema;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -78,23 +78,23 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ModelServiceImpl implements ModelService {
 
-    private ModelRepository modelRepository;
+    private final ModelRepository modelRepository;
 
-    private DatabaseService databaseService;
+    private final DatabaseService databaseService;
 
-    private DimensionService dimensionService;
+    private final DimensionService dimensionService;
 
-    private MetricService metricService;
+    private final MetricService metricService;
 
-    private DomainService domainService;
+    private final DomainService domainService;
 
-    private UserService userService;
+    private final UserService userService;
 
-    private DataSetService dataSetService;
+    private final DataSetService dataSetService;
 
-    private DateInfoRepository dateInfoRepository;
+    private final DateInfoRepository dateInfoRepository;
 
-    private ModelRelaService modelRelaService;
+    private final ModelRelaService modelRelaService;
 
     ExecutorService executor =
             new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
@@ -292,12 +292,9 @@ public class ModelServiceImpl implements ModelService {
         if (modelReq.getModelDetail() == null) {
             return;
         }
-        List<Dim> dims = modelReq.getModelDetail().getDimensions();
+        List<Dimension> dims = modelReq.getModelDetail().getDimensions();
         List<Measure> measures = modelReq.getModelDetail().getMeasures();
         List<Identify> identifies = modelReq.getModelDetail().getIdentifiers();
-        if (CollectionUtils.isEmpty(dims)) {
-            throw new InvalidArgumentException("缺少维度信息");
-        }
         for (Measure measure : measures) {
             String measureForbiddenCharacters =
                     NameCheckUtils.findForbiddenCharacters(measure.getName());
@@ -308,7 +305,7 @@ public class ModelServiceImpl implements ModelService {
                 throw new InvalidArgumentException(message);
             }
         }
-        for (Dim dim : dims) {
+        for (Dimension dim : dims) {
             String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName());
             if (StringUtils.isNotBlank(dim.getName())
                     && StringUtils.isNotBlank(dimForbiddenCharacters)) {
@@ -337,12 +334,10 @@ public class ModelServiceImpl implements ModelService {
         Set<String> relations = new HashSet<>();
         for (ModelRela modelRela : modelRelas) {
             if (modelRela.getFromModelId().equals(modelReq.getId())) {
-                modelRela.getJoinConditions().stream()
-                        .forEach(r -> relations.add(r.getLeftField()));
+                modelRela.getJoinConditions().forEach(r -> relations.add(r.getLeftField()));
             }
             if (modelRela.getToModelId().equals(modelReq.getId())) {
-                modelRela.getJoinConditions().stream()
-                        .forEach(r -> relations.add(r.getRightField()));
+                modelRela.getJoinConditions().forEach(r -> relations.add(r.getRightField()));
             }
         }
         if (relations.isEmpty()) {
@@ -351,10 +346,10 @@ public class ModelServiceImpl implements ModelService {
         // any identify in model relation should not be deleted
         if (modelReq.getModelDetail() == null
                 || CollectionUtils.isEmpty(modelReq.getModelDetail().getIdentifiers())) {
-            throw new InvalidArgumentException(String.format("模型关联中主键/外键不存在, 请检查"));
+            throw new InvalidArgumentException("模型关联中主键/外键不存在, 请检查");
         }
         List<String> modelIdentifiers = modelReq.getModelDetail().getIdentifiers().stream()
-                .map(i -> i.getBizName()).collect(Collectors.toList());
+                .map(Identify::getBizName).collect(Collectors.toList());
         for (String rela : relations) {
             if (!modelIdentifiers.contains(rela)) {
                 throw new InvalidArgumentException(String.format("模型关联中主键/外键(%s)不存在, 请检查", rela));
@@ -459,7 +454,7 @@ public class ModelServiceImpl implements ModelService {
         }
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setDomainIds(domainIds);
-        modelFilter.setIncludesDetail(false);
+        modelFilter.setIncludesDetail(true);
         List<ModelResp> modelResps = getModelList(modelFilter);
         if (CollectionUtils.isEmpty(modelResps)) {
             return modelResps;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service.impl;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
@@ -93,7 +93,7 @@ public class TagQueryServiceImpl implements TagQueryService {
     private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user)
             throws Exception {
         ModelResp model = modelService.getModel(tag.getModelId());
-        List<Dim> timeDimension = model.getTimeDimension();
+        List<Dimension> timeDimension = model.getTimeDimension();
         if (CollectionUtils.isEmpty(timeDimension)) {
             itemValueReq.setDateConf(null);
             return;
@@ -112,12 +112,12 @@ public class TagQueryServiceImpl implements TagQueryService {
         itemValueReq.setDateConf(dateConf);
     }
 
-    private String queryTagDate(Dim dim) {
+    private String queryTagDate(Dimension dim) {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dim.getDateFormat());
         return LocalDate.now().plusDays(-dayBefore).format(formatter);
     }
 
-    private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq,
+    private String queryTagDateFromDbBySql(Dimension dim, TagResp tag, ItemValueReq itemValueReq,
             User user) {
 
         String sqlPattern = "select max(%s)  as %s from tbl where %s is not null";
@@ -129,7 +129,7 @@ public class TagQueryServiceImpl implements TagQueryService {
         if (Objects.nonNull(itemValueReq) && itemValueReq.getDateConf().getUnit() > 1) {
             ModelResp model = modelService.getModel(tag.getModelId());
             if (Objects.nonNull(model)) {
-                List<Dim> timeDims = model.getTimeDimension();
+                List<Dimension> timeDims = model.getTimeDimension();
                 if (!CollectionUtils.isEmpty(timeDims)) {
                     String dateFormat = timeDims.get(0).getDateFormat();
                     if (StringUtils.isEmpty(dateFormat)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java

@@ -69,7 +69,7 @@ public class MetaEmbeddingTask implements CommandLineRunner {
             embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
                     TextSegmentConvert.convertToEmbedding(metricDataItems));
 
-            List<DataItem> dimensionDataItems = dimensionService.getDataEvent().getDataItems();
+            List<DataItem> dimensionDataItems = dimensionService.getAllDataEvents().getDataItems();
             embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
                     TextSegmentConvert.convertToEmbedding(dimensionDataItems));
         } catch (Exception e) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -14,7 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -401,7 +401,7 @@ public class DictUtils {
     private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model,
             Integer itemValueDateStart, Integer itemValueDateEnd) {
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 DateConf dateConf = new DateConf();
                 dateConf.setDateMode(DateConf.DateMode.BETWEEN);
@@ -496,7 +496,7 @@ public class DictUtils {
     private boolean partitionedModel(Long modelId) {
         ModelResp model = modelService.getModel(modelId);
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 return true;
             }
@@ -507,7 +507,7 @@ public class DictUtils {
     private String generateDictDateFilterRecent(DictItemResp dictItemResp) {
         ModelResp model = modelService.getModel(dictItemResp.getModelId());
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 String dateFormat = timeDims.get(0).getDateFormat();
                 if (StringUtils.isEmpty(dateFormat)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -105,6 +105,8 @@ public class DimensionConverter {
         dimensionResp.setType(getType(dimensionDO.getType()));
         dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
         dimensionResp.setIsTag(dimensionDO.getIsTag());
+        dimensionResp.setDomainId(modelRespMap
+                .getOrDefault(dimensionResp.getModelId(), new ModelResp()).getDomainId());
         return dimensionResp;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java

@@ -7,7 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.PublishEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
@@ -82,7 +82,7 @@ public class MetricConverter {
             metricResp.setModelName(modelResp.getName());
             metricResp.setModelBizName(modelResp.getBizName());
             metricResp.setDomainId(modelResp.getDomainId());
-            List<Dim> timeDims = modelResp.getTimeDimension();
+            List<Dimension> timeDims = modelResp.getTimeDimension();
             if (CollectionUtils.isNotEmpty(timeDims)) {
                 metricResp.setContainsPartitionDimensions(true);
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -111,7 +111,7 @@ public class ModelConverter {
         return measureResp;
     }
 
-    public static DimensionReq convert(Dim dim, ModelDO modelDO) {
+    public static DimensionReq convert(Dimension dim, ModelDO modelDO) {
         DimensionReq dimensionReq = new DimensionReq();
         dimensionReq.setName(dim.getName());
         dimensionReq.setBizName(dim.getBizName());
@@ -129,7 +129,6 @@ public class ModelConverter {
         dimensionReq.setType(dim.getType().name());
         dimensionReq
                 .setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
-        dimensionReq.setIsTag(dim.getIsTag());
         dimensionReq.setTypeParams(dim.getTypeParams());
         return dimensionReq;
     }
@@ -165,8 +164,8 @@ public class ModelConverter {
     public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq,
             String tableName) {
         ModelReq modelReq = new ModelReq();
-        modelReq.setName(modelSchema.getName());
-        modelReq.setBizName(modelSchema.getBizName());
+        modelReq.setName(modelBuildReq.getName());
+        modelReq.setBizName(modelBuildReq.getBizName());
         modelReq.setDatabaseId(modelBuildReq.getDatabaseId());
         modelReq.setDomainId(modelBuildReq.getDomainId());
         ModelDetail modelDetail = new ModelDetail();
@@ -188,7 +187,7 @@ public class ModelConverter {
                         columnSchema.getAgg().getOperator(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dim dim = new Dim(columnSchema.getName(), columnSchema.getColumnName(),
+                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
                         DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }
@@ -198,10 +197,12 @@ public class ModelConverter {
     }
 
     private static IdentifyType getIdentifyType(FieldType fieldType) {
-        if (FieldType.foreign_key.equals(fieldType) || FieldType.primary_key.equals(fieldType)) {
+        if (FieldType.primary_key.equals(fieldType)) {
             return IdentifyType.primary;
-        } else {
+        } else if (FieldType.foreign_key.equals(fieldType)) {
             return IdentifyType.foreign;
+        } else {
+            return null;
         }
     }
 
@@ -214,7 +215,7 @@ public class ModelConverter {
         return modelDescs;
     }
 
-    private static boolean isCreateDimension(Dim dim) {
+    private static boolean isCreateDimension(Dimension dim) {
         return dim.getIsCreateDimension() == 1 && StringUtils.isNotBlank(dim.getName());
     }
 
@@ -226,7 +227,7 @@ public class ModelConverter {
         return measure.getIsCreateMetric() == 1 && StringUtils.isNotBlank(measure.getName());
     }
 
-    public static List<Dim> getDimToCreateDimension(ModelDetail modelDetail) {
+    public static List<Dimension> getDimToCreateDimension(ModelDetail modelDetail) {
         if (CollectionUtils.isEmpty(modelDetail.getDimensions())) {
             return Lists.newArrayList();
         }
@@ -254,7 +255,7 @@ public class ModelConverter {
         List<DimensionReq> dimensionReqs = Lists.newArrayList();
         ModelDetail modelDetail =
                 JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class);
-        List<Dim> dims = getDimToCreateDimension(modelDetail);
+        List<Dimension> dims = getDimToCreateDimension(modelDetail);
         if (!CollectionUtils.isEmpty(dims)) {
             dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName()))
                     .map(dim -> convert(dim, modelDO)).collect(Collectors.toList());

headless/server/src/main/resources/mapper/StatMapper.xml

@@ -40,28 +40,4 @@
         <result column="query_opt_mode" property="queryOptMode"/>
     </resultMap>
 
-    <select id="getStatInfo"
-            resultType="com.tencent.supersonic.headless.api.pojo.QueryStat">
-        select *
-        from s2_query_stat_info
-        <where>
-            <if test="startTime != null">
-                and start_time >= #{startTime}String.valueOf(queryFilter.getValue())
-            </if>
-            <if test="modelId != null">
-                and model_id = #{modelId}
-            </if>
-            <if test="modelIds != null and modelIds.size() > 0">
-                and model_id in
-                <foreach item="id" collection="modelIds" open="(" separator="," close=")">
-                    #{id}
-                </foreach>
-            </if>
-            <if test="metric != null">
-                and metrics like concat('%',#{metric},'%')
-            </if>
-        </where>
-
-    </select>
-
 </mapper>

headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml

@@ -116,63 +116,6 @@
         </foreach>
     </update>
 
-    <select id="query" resultMap="ResultMapWithBLOBs">
-        select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
-        from s2_dimension t
-        left join (
-             select *
-             from s2_tag
-             where type = 'DIMENSION'
-        ) t1 on t.id = t1.item_id
-        where status != 3
-        <if test="key != null and key != ''">
-            and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
-        </if>
-        <if test="id != null">
-            and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="name != null and name != '' ">
-            and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="bizName != null and bizName != ''">
-            and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="sensitiveLevel != null">
-            and t.sensitive_level = #{sensitiveLevel}
-        </if>
-        <if test="status != null">
-            and t.status = #{status}
-        </if>
-        <if test="modelIds != null and modelIds.size >0">
-            and t.model_id in
-            <foreach collection="modelIds" index="index" item="model" open="(" close=")"
-                     separator=",">
-                #{model}
-            </foreach>
-        </if>
-        <if test="ids != null and ids.size >0">
-            and t.id in
-            <foreach collection="ids" index="index" item="id" open="(" close=")"
-                     separator=",">
-                #{id}
-            </foreach>
-        </if>
-        <if test="createdBy != null">
-            and t.created_by = #{createdBy}
-        </if>
-        <if test="isTag != null and isTag == 1">
-            and t1.id is not null
-        </if>
-        <if test="isTag != null and isTag == 0">
-            and t1.id is null
-        </if>
-    </select>
-
-
     <select id="queryDimensions" resultMap="ResultMapWithBLOBs">
         select *
         from s2_dimension

headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml

@@ -127,70 +127,6 @@
         </foreach>
     </update>
 
-    <select id="query" resultMap="ResultMapWithBLOBs">
-        select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
-        from s2_metric t
-        left join (
-           select *
-           from s2_tag
-           where type = 'METRIC'
-        ) t1 on t.id = t1.item_id
-        where t.status != 3
-        <if test="type != null and type != ''">
-            and t.type = #{type}
-        </if>
-        <if test="key != null and key != ''">
-            and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.classifications like CONCAT('%',#{key , jdbcType=VARCHAR},'%')  or
-            t.created_by like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
-        </if>
-        <if test="id != null">
-            and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="name != null and name != '' ">
-            and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="bizName != null and bizName != ''">
-            and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="sensitiveLevel != null">
-            and t.sensitive_level = #{sensitiveLevel}
-        </if>
-        <if test="status != null">
-            and t.status = #{status}
-        </if>
-        <if test="modelIds != null and modelIds.size >0">
-            and t.model_id in
-            <foreach collection="modelIds" index="index" item="model" open="(" close=")"
-                    separator=",">
-                #{model}
-            </foreach>
-        </if>
-        <if test="ids != null and ids.size >0">
-            and t.id in
-            <foreach collection="ids" index="index" item="id" open="(" close=")"
-                    separator=",">
-                #{id}
-            </foreach>
-        </if>
-        <if test="createdBy != null">
-            and t.created_by = #{createdBy}
-        </if>
-        <if test="isTag != null and isTag == 1">
-            and t1.id is not null
-        </if>
-        <if test="isTag != null and isTag == 0">
-            and t1.id is null
-        </if>
-        <if test="isPublish != null and isPublish == 1">
-           and (t.created_by = #{userName} or t.is_publish = 1)
-        </if>
-    </select>
-
     <select id="queryMetrics" resultMap="ResultMapWithBLOBs">
         select *
         from s2_metric

headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -95,11 +95,11 @@ class ModelServiceImplTest {
         List<Identify> identifiers = new ArrayList<>();
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -134,11 +134,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名_a", IdentifyType.primary.name(), "user_name_a"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date_a", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date_a", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page_a", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page_a", DimensionType.categorical, 0);
         dimension2.setExpr("page_a");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -169,11 +169,11 @@ class ModelServiceImplTest {
         List<Identify> identifiers = new ArrayList<>();
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -207,11 +207,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -252,11 +252,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名_a", IdentifyType.primary.name(), "user_name_a"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date_a", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date_a", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page_a", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page_a", DimensionType.categorical, 0);
         dimension2.setExpr("page_a");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);