[improvement]Use QueryWrapper in place of hard-coded SQLs (#1944)

* [improvement][launcher]Use API to get element ID avoiding hard-code.

* [fix][launcher]Fix mysql scripts.

* [improvement][launcher]Support DuckDB database and refactor translator code structure.

* [improvement][headless-fe] Revamped the interaction for semantic modeling routing and successfully implemented the switching between dimension and dataset management.

* [improvement][Headless] Add table ddl in Dbschema

* [improvement][Headless] Add get database by type

* [improvement][Headless] Supports automatic batch creation of models based on db table names.

* [improvement][Headless] Supports getting domain by bizName

* [improvement][launcher]Refactor unit tests and demo data.

* [fix][launcher]Change default vector dimension to 512.

* [improvement](Dict) add dimValueAliasMap info for KnowledgeBaseService

* [improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml.

* [improvement][chat]Introduce ChatMemory to delegate ChatMemoryDO.

* [fix][common]Fix embedding store sys configs.

* [fix][common]Fix postgres schema, using varchar instead of char.

* [improvement][launcher]Change supersonic docker deployment from mysql to postgres.

* [Fix][launcher]Fix a number of issues related to semantic modeling.

* [Fix][headless]Fix the evaluation logic of agg type.

* [fix][assembly]Fix Dockerfile and add docker compose run script.

* [fix][chat]Fix "multiple assignments to same column "similar_queries".

* [improvement][headless]Use LamdaQueryWrapper to avoid hard-coded column names.

* [improvement][headless]Refactor headless infra to support advanced semantic modelling.

* [improvement][headless]Change class name `Dim` to `Dimension`.

* [improvement][chat]Introduce `TimeFieldMapper` to always map time field.

* [fix][headless]Remove unnecessary dimension existence check.

* [fix][chat]Fix adjusted filters don't take effect.

---------

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -207,7 +207,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
         ModelResp modelResp = modelResps.get(0);
         String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
                 modelResp.getName());
-        List<Dim> timeDims = modelResp.getTimeDimension();
+        List<Dimension> timeDims = modelResp.getTimeDimension();
         if (CollectionUtils.isNotEmpty(timeDims)) {
             sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
                     TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),

headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java

@@ -27,7 +27,7 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
             dictWord.setWord(dataItem.getName());
             String sign = DictWordType.NATURE_SPILT;
             String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
-            String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
+            String nature = sign + dataItem.getModelId() + sign + dataItem.getId() + suffixNature;
             String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
             dictWord.setNature(nature);
             dictWord.setNatureWithFrequency(natureWithFrequency);

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.headless.server.manager;
 
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
@@ -18,7 +18,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -54,7 +53,7 @@ public class ModelYamlManager {
         return dataModelYamlTpl;
     }
 
-    public static DimensionYamlTpl convert(Dim dim) {
+    public static DimensionYamlTpl convert(Dimension dim) {
         DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
         BeanUtils.copyProperties(dim, dimensionYamlTpl);
         dimensionYamlTpl.setName(dim.getBizName());
@@ -85,15 +84,4 @@ public class ModelYamlManager {
         return identifyYamlTpl;
     }
 
-    private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) {
-        Measure measure = new Measure();
-        measure.setExpr("1");
-        if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
-            measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
-        }
-        measure.setAgg("count");
-        measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
-        measure.setIsCreateMetric(1);
-        datasourceDetail.getMeasures().add(measure);
-    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -23,7 +23,7 @@ public class RuleSemanticModeller implements SemanticModeller {
 
     private ColumnSchema convert(DBColumn dbColumn) {
         ColumnSchema columnSchema = new ColumnSchema();
-        columnSchema.setName(dbColumn.getComment());
+        columnSchema.setName(dbColumn.getColumnName());
         columnSchema.setColumnName(dbColumn.getColumnName());
         columnSchema.setComment(dbColumn.getComment());
         columnSchema.setDataType(dbColumn.getDataType());

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java

@@ -16,6 +16,7 @@ public class ModelDO {
 
     private Long domainId;
 
+    @Deprecated
     private Long tagObjectId;
 
     private String name;

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryStatDO.java

@@ -28,6 +28,7 @@ public class QueryStatDO {
     private String queryStructCmd;
     @TableField("struct_cmd_md5")
     private String queryStructCmdMd5;
+    @TableField("\"sql\"")
     private String sql;
     private String sqlMd5;
     private String queryEngine;

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
-import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import org.apache.ibatis.annotations.Mapper;
 
@@ -12,11 +11,7 @@ public interface DimensionDOCustomMapper {
 
     void batchInsert(List<DimensionDO> dimensionDOS);
 
-    void batchUpdate(List<DimensionDO> dimensionDOS);
-
     void batchUpdateStatus(List<DimensionDO> dimensionDOS);
 
-    List<DimensionDO> query(DimensionFilter dimensionFilter);
-
     List<DimensionDO> queryDimensions(DimensionsFilter dimensionsFilter);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
-import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
 import org.apache.ibatis.annotations.Mapper;
 
@@ -20,7 +19,5 @@ public interface MetricDOCustomMapper {
 
     void updateClassificationsBatch(List<MetricDO> metricDOS);
 
-    List<MetricDO> query(MetricFilter metricFilter);
-
     List<MetricDO> queryMetrics(MetricsFilter metricsFilter);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java

@@ -1,15 +1,9 @@
 package com.tencent.supersonic.headless.server.persistence.mapper;
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.tencent.supersonic.headless.api.pojo.QueryStat;
-import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.server.persistence.dataobject.QueryStatDO;
 import org.apache.ibatis.annotations.Mapper;
 
-import java.util.List;
-
 @Mapper
 public interface StatMapper extends BaseMapper<QueryStatDO> {
-
-    List<QueryStat> getStatInfo(ItemUseReq itemUseCommend);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -118,6 +118,7 @@ public class DictRepositoryImpl implements DictRepository {
             wrapper.lambda().and(qw -> qw.like(DictTaskDO::getName, key).or()
                     .like(DictTaskDO::getDescription, key).or().like(DictTaskDO::getConfig, key));
         }
+        wrapper.lambda().orderByDesc(DictTaskDO::getCreatedAt);
         return dictTaskMapper.selectList(wrapper);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java

@@ -1,14 +1,17 @@
 package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCustomMapper;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Objects;
 
 @Service
 public class DimensionRepositoryImpl implements DimensionRepository {
@@ -50,7 +53,43 @@ public class DimensionRepositoryImpl implements DimensionRepository {
 
     @Override
     public List<DimensionDO> getDimension(DimensionFilter dimensionFilter) {
-        return dimensionDOCustomMapper.query(dimensionFilter);
+        QueryWrapper<DimensionDO> queryWrapper = new QueryWrapper<>();
+        queryWrapper.lambda().ne(DimensionDO::getStatus, 3);
+        if (Objects.nonNull(dimensionFilter.getIds()) && !dimensionFilter.getIds().isEmpty()) {
+            queryWrapper.lambda().in(DimensionDO::getId, dimensionFilter.getIds());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getId())) {
+            queryWrapper.lambda().eq(DimensionDO::getId, dimensionFilter.getId());
+        }
+        if (Objects.nonNull(dimensionFilter.getModelIds())
+                && !dimensionFilter.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(DimensionDO::getModelId, dimensionFilter.getModelIds());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getName())) {
+            queryWrapper.lambda().like(DimensionDO::getName, dimensionFilter.getName());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getId())) {
+            queryWrapper.lambda().like(DimensionDO::getBizName, dimensionFilter.getBizName());
+        }
+        if (Objects.nonNull(dimensionFilter.getStatus())) {
+            queryWrapper.lambda().eq(DimensionDO::getStatus, dimensionFilter.getStatus());
+        }
+        if (Objects.nonNull(dimensionFilter.getSensitiveLevel())) {
+            queryWrapper.lambda().eq(DimensionDO::getSensitiveLevel,
+                    dimensionFilter.getSensitiveLevel());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getCreatedBy())) {
+            queryWrapper.lambda().eq(DimensionDO::getCreatedBy, dimensionFilter.getCreatedBy());
+        }
+        if (StringUtils.isNotBlank(dimensionFilter.getKey())) {
+            String key = dimensionFilter.getKey();
+            queryWrapper.lambda().like(DimensionDO::getName, key).or()
+                    .like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key)
+                    .or().like(DimensionDO::getAlias, key).or()
+                    .like(DimensionDO::getCreatedBy, key);
+        }
+
+        return dimensionDOMapper.selectList(queryWrapper);
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java

@@ -9,9 +9,11 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefa
 import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
 import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.Objects;
 
 @Component
 public class MetricRepositoryImpl implements MetricRepository {
@@ -73,7 +75,46 @@ public class MetricRepositoryImpl implements MetricRepository {
 
     @Override
     public List<MetricDO> getMetric(MetricFilter metricFilter) {
-        return metricDOCustomMapper.query(metricFilter);
+        QueryWrapper<MetricDO> queryWrapper = new QueryWrapper<>();
+        queryWrapper.lambda().ne(MetricDO::getStatus, 3);
+        if (Objects.nonNull(metricFilter.getIds()) && !metricFilter.getIds().isEmpty()) {
+            queryWrapper.lambda().in(MetricDO::getId, metricFilter.getIds());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getId())) {
+            queryWrapper.lambda().eq(MetricDO::getId, metricFilter.getId());
+        }
+        if (Objects.nonNull(metricFilter.getModelIds()) && !metricFilter.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(MetricDO::getModelId, metricFilter.getModelIds());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getType())) {
+            queryWrapper.lambda().eq(MetricDO::getType, metricFilter.getType());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getName())) {
+            queryWrapper.lambda().like(MetricDO::getName, metricFilter.getName());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getId())) {
+            queryWrapper.lambda().like(MetricDO::getBizName, metricFilter.getBizName());
+        }
+        if (Objects.nonNull(metricFilter.getStatus())) {
+            queryWrapper.lambda().eq(MetricDO::getStatus, metricFilter.getStatus());
+        }
+        if (Objects.nonNull(metricFilter.getSensitiveLevel())) {
+            queryWrapper.lambda().eq(MetricDO::getSensitiveLevel, metricFilter.getSensitiveLevel());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) {
+            queryWrapper.lambda().eq(MetricDO::getCreatedBy, metricFilter.getCreatedBy());
+        }
+        if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) {
+            queryWrapper.lambda().eq(MetricDO::getIsPublish, metricFilter.getIsPublish());
+        }
+        if (StringUtils.isNotBlank(metricFilter.getKey())) {
+            String key = metricFilter.getKey();
+            queryWrapper.lambda().like(MetricDO::getName, key).or().like(MetricDO::getBizName, key)
+                    .or().like(MetricDO::getDescription, key).or().like(MetricDO::getAlias, key)
+                    .or().like(MetricDO::getCreatedBy, key);
+        }
+
+        return metricDOMapper.selectList(queryWrapper);
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java

@@ -51,7 +51,7 @@ public class QueryRuleRepositoryImpl implements QueryRuleRepository {
 
         QueryWrapper<QueryRuleDO> wrapperSys = new QueryWrapper<>();
         // 返回系统设置的规则
-        wrapperSys.or().eq("priority", 0L);
+        wrapperSys.lambda().or().eq(QueryRuleDO::getPriority, 0L);
         List<QueryRuleDO> queryRuleDOListSys = mapper.selectList(wrapperSys);
 
         queryRuleDOList.addAll(queryRuleDOListSys);

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -47,7 +48,7 @@ public class StatRepositoryImpl implements StatRepository {
     @SneakyThrows
     public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
         List<ItemUseResp> result = new ArrayList<>();
-        List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
+        List<QueryStatDO> statInfos = getQueryStats(itemUseReq);
         Map<String, Long> map = new ConcurrentHashMap<>();
         statInfos.stream().forEach(stat -> {
             String dimensions = stat.getDimensions();
@@ -70,6 +71,21 @@ public class StatRepositoryImpl implements StatRepository {
                 .collect(Collectors.toList());
     }
 
+    private List<QueryStatDO> getQueryStats(ItemUseReq itemUseReq) {
+        QueryWrapper<QueryStatDO> queryWrapper = new QueryWrapper<>();
+        if (Objects.nonNull(itemUseReq.getModelId())) {
+            queryWrapper.lambda().eq(QueryStatDO::getModelId, itemUseReq.getModelId());
+        }
+        if (Objects.nonNull(itemUseReq.getModelIds()) && !itemUseReq.getModelIds().isEmpty()) {
+            queryWrapper.lambda().in(QueryStatDO::getModelId, itemUseReq.getModelIds());
+        }
+        if (Objects.nonNull(itemUseReq.getMetric())) {
+            queryWrapper.lambda().like(QueryStatDO::getMetrics, itemUseReq.getMetric());
+        }
+
+        return statMapper.selectList(queryWrapper);
+    }
+
     private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type,
             Long dataSetId) {
         if (StringUtils.isNotEmpty(dimensions)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java

@@ -47,7 +47,7 @@ public interface DimensionService {
 
     void sendDimensionEventBatch(List<Long> modelIds, EventType eventType);
 
-    DataEvent getDataEvent();
+    DataEvent getAllDataEvents();
 
     Boolean updateDimValueAlias(DimValueAliasReq req, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java

@@ -79,7 +79,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
         DataSetDO dataSetDO = convert(dataSetReq);
         dataSetDO.setStatus(StatusEnum.ONLINE.getCode());
         DataSetResp dataSetResp = convert(dataSetDO);
-        conflictCheck(dataSetResp);
+        // conflictCheck(dataSetResp);
         save(dataSetDO);
         dataSetResp.setId(dataSetDO.getId());
         return dataSetResp;
@@ -90,7 +90,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
         dataSetReq.updatedBy(user.getName());
         DataSetDO dataSetDO = convert(dataSetReq);
         DataSetResp dataSetResp = convert(dataSetDO);
-        conflictCheck(dataSetResp);
+        // conflictCheck(dataSetResp);
         updateById(dataSetDO);
         return dataSetResp;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -225,6 +225,9 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
         if (StringUtils.isNotBlank(modelBuildReq.getSql())) {
             List<DBColumn> columns =
                     getColumns(modelBuildReq.getDatabaseId(), modelBuildReq.getSql());
+            DatabaseResp databaseResp = getDatabase(modelBuildReq.getDatabaseId());
+            DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
+            columns.forEach(c -> c.setFieldType(engineAdaptor.classifyColumnType(c.getDataType())));
             dbColumnMap.put(modelBuildReq.getSql(), columns);
         } else {
             for (String table : modelBuildReq.getTables()) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -6,12 +6,9 @@ import com.alibaba.fastjson.TypeReference;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DataEvent;
-import com.tencent.supersonic.common.pojo.DataItem;
-import com.tencent.supersonic.common.pojo.ModelRela;
-import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -25,23 +22,14 @@ import com.tencent.supersonic.headless.api.pojo.request.DimValueAliasReq;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
-import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import com.tencent.supersonic.headless.server.pojo.ModelFilter;
-import com.tencent.supersonic.headless.server.service.DataSetService;
-import com.tencent.supersonic.headless.server.service.DatabaseService;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.ModelRelaService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.service.*;
 import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
 import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
@@ -152,11 +140,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         DimensionConverter.convert(dimensionDO, dimensionReq);
         dimensionRepository.updateDimension(dimensionDO);
         if (!oldName.equals(dimensionDO.getName())) {
-            sendEvent(
-                    DataItem.builder().modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
-                            .newName(dimensionReq.getName()).name(oldName).type(TypeEnums.DIMENSION)
-                            .id(dimensionDO.getId() + Constants.UNDERLINE).build(),
-                    EventType.UPDATE);
+            sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
         }
     }
 
@@ -424,7 +408,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         eventPublisher.publishEvent(dataEvent);
     }
 
-    public DataEvent getDataEvent() {
+    public DataEvent getAllDataEvents() {
         DimensionFilter dimensionFilter = new DimensionFilter();
         List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
         return getDataEvent(dimensionDOS, EventType.ADD);
@@ -464,13 +448,18 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         return true;
     }
 
+    private DataItem getDataItem(DimensionDO dimensionDO) {
+        ModelResp modelResp = modelService.getModel(dimensionDO.getModelId());
+        DimensionResp dimensionResp = DimensionConverter.convert2DimensionResp(dimensionDO,
+                ImmutableMap.of(modelResp.getId(), modelResp));
+        return DataItem.builder().id(dimensionResp.getId().toString()).name(dimensionResp.getName())
+                .bizName(dimensionResp.getBizName()).modelId(dimensionResp.getModelId().toString())
+                .domainId(dimensionResp.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
+    }
+
     private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
-        List<DataItem> dataItems = dimensionDOS.stream()
-                .map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE)
-                        .name(dimensionDO.getName())
-                        .modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
-                        .type(TypeEnums.DIMENSION).build())
-                .collect(Collectors.toList());
+        List<DataItem> dataItems =
+                dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList());
         return new DataEvent(this, dataItems, eventType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -5,59 +5,26 @@ import com.alibaba.fastjson.TypeReference;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DataEvent;
-import com.tencent.supersonic.common.pojo.DataItem;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.*;
 import com.tencent.supersonic.common.pojo.enums.*;
 import com.tencent.supersonic.common.util.BeanMapper;
-import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
-import com.tencent.supersonic.headless.api.pojo.Measure;
-import com.tencent.supersonic.headless.api.pojo.MeasureParam;
-import com.tencent.supersonic.headless.api.pojo.MetaFilter;
-import com.tencent.supersonic.headless.api.pojo.MetricParam;
-import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
-import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
-import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
-import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.request.*;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
-import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
-import com.tencent.supersonic.headless.server.pojo.MetricFilter;
-import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
-import com.tencent.supersonic.headless.server.pojo.ModelCluster;
-import com.tencent.supersonic.headless.server.pojo.ModelFilter;
-import com.tencent.supersonic.headless.server.service.CollectService;
-import com.tencent.supersonic.headless.server.service.DataSetService;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.pojo.*;
+import com.tencent.supersonic.headless.server.service.*;
 import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
 import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -70,18 +37,7 @@ import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 @Service
@@ -667,12 +623,13 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
     }
 
     private DataItem getDataItem(MetricDO metricDO) {
-        MetricResp metricResp =
-                MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
+        ModelResp modelResp = modelService.getModel(metricDO.getModelId());
+        MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO,
+                ImmutableMap.of(modelResp.getId(), modelResp), Lists.newArrayList());
         fillDefaultAgg(metricResp);
-        return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE)
-                .name(metricDO.getName()).bizName(metricDO.getBizName())
-                .modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC)
+        return DataItem.builder().id(metricResp.getId().toString()).name(metricResp.getName())
+                .bizName(metricResp.getBizName()).modelId(metricResp.getModelId().toString())
+                .domainId(metricResp.getDomainId().toString()).type(TypeEnums.METRIC)
                 .defaultAgg(metricResp.getDefaultAgg()).build();
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.DbSchema;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -78,23 +78,23 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ModelServiceImpl implements ModelService {
 
-    private ModelRepository modelRepository;
+    private final ModelRepository modelRepository;
 
-    private DatabaseService databaseService;
+    private final DatabaseService databaseService;
 
-    private DimensionService dimensionService;
+    private final DimensionService dimensionService;
 
-    private MetricService metricService;
+    private final MetricService metricService;
 
-    private DomainService domainService;
+    private final DomainService domainService;
 
-    private UserService userService;
+    private final UserService userService;
 
-    private DataSetService dataSetService;
+    private final DataSetService dataSetService;
 
-    private DateInfoRepository dateInfoRepository;
+    private final DateInfoRepository dateInfoRepository;
 
-    private ModelRelaService modelRelaService;
+    private final ModelRelaService modelRelaService;
 
     ExecutorService executor =
             new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
@@ -292,12 +292,9 @@ public class ModelServiceImpl implements ModelService {
         if (modelReq.getModelDetail() == null) {
             return;
         }
-        List<Dim> dims = modelReq.getModelDetail().getDimensions();
+        List<Dimension> dims = modelReq.getModelDetail().getDimensions();
         List<Measure> measures = modelReq.getModelDetail().getMeasures();
         List<Identify> identifies = modelReq.getModelDetail().getIdentifiers();
-        if (CollectionUtils.isEmpty(dims)) {
-            throw new InvalidArgumentException("缺少维度信息");
-        }
         for (Measure measure : measures) {
             String measureForbiddenCharacters =
                     NameCheckUtils.findForbiddenCharacters(measure.getName());
@@ -308,7 +305,7 @@ public class ModelServiceImpl implements ModelService {
                 throw new InvalidArgumentException(message);
             }
         }
-        for (Dim dim : dims) {
+        for (Dimension dim : dims) {
             String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName());
             if (StringUtils.isNotBlank(dim.getName())
                     && StringUtils.isNotBlank(dimForbiddenCharacters)) {
@@ -337,12 +334,10 @@ public class ModelServiceImpl implements ModelService {
         Set<String> relations = new HashSet<>();
         for (ModelRela modelRela : modelRelas) {
             if (modelRela.getFromModelId().equals(modelReq.getId())) {
-                modelRela.getJoinConditions().stream()
-                        .forEach(r -> relations.add(r.getLeftField()));
+                modelRela.getJoinConditions().forEach(r -> relations.add(r.getLeftField()));
             }
             if (modelRela.getToModelId().equals(modelReq.getId())) {
-                modelRela.getJoinConditions().stream()
-                        .forEach(r -> relations.add(r.getRightField()));
+                modelRela.getJoinConditions().forEach(r -> relations.add(r.getRightField()));
             }
         }
         if (relations.isEmpty()) {
@@ -351,10 +346,10 @@ public class ModelServiceImpl implements ModelService {
         // any identify in model relation should not be deleted
         if (modelReq.getModelDetail() == null
                 || CollectionUtils.isEmpty(modelReq.getModelDetail().getIdentifiers())) {
-            throw new InvalidArgumentException(String.format("模型关联中主键/外键不存在, 请检查"));
+            throw new InvalidArgumentException("模型关联中主键/外键不存在, 请检查");
         }
         List<String> modelIdentifiers = modelReq.getModelDetail().getIdentifiers().stream()
-                .map(i -> i.getBizName()).collect(Collectors.toList());
+                .map(Identify::getBizName).collect(Collectors.toList());
         for (String rela : relations) {
             if (!modelIdentifiers.contains(rela)) {
                 throw new InvalidArgumentException(String.format("模型关联中主键/外键(%s)不存在, 请检查", rela));
@@ -459,7 +454,7 @@ public class ModelServiceImpl implements ModelService {
         }
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setDomainIds(domainIds);
-        modelFilter.setIncludesDetail(false);
+        modelFilter.setIncludesDetail(true);
         List<ModelResp> modelResps = getModelList(modelFilter);
         if (CollectionUtils.isEmpty(modelResps)) {
             return modelResps;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service.impl;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
@@ -93,7 +93,7 @@ public class TagQueryServiceImpl implements TagQueryService {
     private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user)
             throws Exception {
         ModelResp model = modelService.getModel(tag.getModelId());
-        List<Dim> timeDimension = model.getTimeDimension();
+        List<Dimension> timeDimension = model.getTimeDimension();
         if (CollectionUtils.isEmpty(timeDimension)) {
             itemValueReq.setDateConf(null);
             return;
@@ -112,12 +112,12 @@ public class TagQueryServiceImpl implements TagQueryService {
         itemValueReq.setDateConf(dateConf);
     }
 
-    private String queryTagDate(Dim dim) {
+    private String queryTagDate(Dimension dim) {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dim.getDateFormat());
         return LocalDate.now().plusDays(-dayBefore).format(formatter);
     }
 
-    private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq,
+    private String queryTagDateFromDbBySql(Dimension dim, TagResp tag, ItemValueReq itemValueReq,
             User user) {
 
         String sqlPattern = "select max(%s)  as %s from tbl where %s is not null";
@@ -129,7 +129,7 @@ public class TagQueryServiceImpl implements TagQueryService {
         if (Objects.nonNull(itemValueReq) && itemValueReq.getDateConf().getUnit() > 1) {
             ModelResp model = modelService.getModel(tag.getModelId());
             if (Objects.nonNull(model)) {
-                List<Dim> timeDims = model.getTimeDimension();
+                List<Dimension> timeDims = model.getTimeDimension();
                 if (!CollectionUtils.isEmpty(timeDims)) {
                     String dateFormat = timeDims.get(0).getDateFormat();
                     if (StringUtils.isEmpty(dateFormat)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java

@@ -69,7 +69,7 @@ public class MetaEmbeddingTask implements CommandLineRunner {
             embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
                     TextSegmentConvert.convertToEmbedding(metricDataItems));
 
-            List<DataItem> dimensionDataItems = dimensionService.getDataEvent().getDataItems();
+            List<DataItem> dimensionDataItems = dimensionService.getAllDataEvents().getDataItems();
             embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
                     TextSegmentConvert.convertToEmbedding(dimensionDataItems));
         } catch (Exception e) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -14,7 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -401,7 +401,7 @@ public class DictUtils {
     private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model,
             Integer itemValueDateStart, Integer itemValueDateEnd) {
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 DateConf dateConf = new DateConf();
                 dateConf.setDateMode(DateConf.DateMode.BETWEEN);
@@ -496,7 +496,7 @@ public class DictUtils {
     private boolean partitionedModel(Long modelId) {
         ModelResp model = modelService.getModel(modelId);
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 return true;
             }
@@ -507,7 +507,7 @@ public class DictUtils {
     private String generateDictDateFilterRecent(DictItemResp dictItemResp) {
         ModelResp model = modelService.getModel(dictItemResp.getModelId());
         if (Objects.nonNull(model)) {
-            List<Dim> timeDims = model.getTimeDimension();
+            List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
                 String dateFormat = timeDims.get(0).getDateFormat();
                 if (StringUtils.isEmpty(dateFormat)) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -105,6 +105,8 @@ public class DimensionConverter {
         dimensionResp.setType(getType(dimensionDO.getType()));
         dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
         dimensionResp.setIsTag(dimensionDO.getIsTag());
+        dimensionResp.setDomainId(modelRespMap
+                .getOrDefault(dimensionResp.getModelId(), new ModelResp()).getDomainId());
         return dimensionResp;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java

@@ -7,7 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.PublishEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
@@ -82,7 +82,7 @@ public class MetricConverter {
             metricResp.setModelName(modelResp.getName());
             metricResp.setModelBizName(modelResp.getBizName());
             metricResp.setDomainId(modelResp.getDomainId());
-            List<Dim> timeDims = modelResp.getTimeDimension();
+            List<Dimension> timeDims = modelResp.getTimeDimension();
             if (CollectionUtils.isNotEmpty(timeDims)) {
                 metricResp.setContainsPartitionDimensions(true);
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -111,7 +111,7 @@ public class ModelConverter {
         return measureResp;
     }
 
-    public static DimensionReq convert(Dim dim, ModelDO modelDO) {
+    public static DimensionReq convert(Dimension dim, ModelDO modelDO) {
         DimensionReq dimensionReq = new DimensionReq();
         dimensionReq.setName(dim.getName());
         dimensionReq.setBizName(dim.getBizName());
@@ -129,7 +129,6 @@ public class ModelConverter {
         dimensionReq.setType(dim.getType().name());
         dimensionReq
                 .setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
-        dimensionReq.setIsTag(dim.getIsTag());
         dimensionReq.setTypeParams(dim.getTypeParams());
         return dimensionReq;
     }
@@ -165,8 +164,8 @@ public class ModelConverter {
     public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq,
             String tableName) {
         ModelReq modelReq = new ModelReq();
-        modelReq.setName(modelSchema.getName());
-        modelReq.setBizName(modelSchema.getBizName());
+        modelReq.setName(modelBuildReq.getName());
+        modelReq.setBizName(modelBuildReq.getBizName());
         modelReq.setDatabaseId(modelBuildReq.getDatabaseId());
         modelReq.setDomainId(modelBuildReq.getDomainId());
         ModelDetail modelDetail = new ModelDetail();
@@ -188,7 +187,7 @@ public class ModelConverter {
                         columnSchema.getAgg().getOperator(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dim dim = new Dim(columnSchema.getName(), columnSchema.getColumnName(),
+                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
                         DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }
@@ -198,10 +197,12 @@ public class ModelConverter {
     }
 
     private static IdentifyType getIdentifyType(FieldType fieldType) {
-        if (FieldType.foreign_key.equals(fieldType) || FieldType.primary_key.equals(fieldType)) {
+        if (FieldType.primary_key.equals(fieldType)) {
             return IdentifyType.primary;
-        } else {
+        } else if (FieldType.foreign_key.equals(fieldType)) {
             return IdentifyType.foreign;
+        } else {
+            return null;
         }
     }
 
@@ -214,7 +215,7 @@ public class ModelConverter {
         return modelDescs;
     }
 
-    private static boolean isCreateDimension(Dim dim) {
+    private static boolean isCreateDimension(Dimension dim) {
         return dim.getIsCreateDimension() == 1 && StringUtils.isNotBlank(dim.getName());
     }
 
@@ -226,7 +227,7 @@ public class ModelConverter {
         return measure.getIsCreateMetric() == 1 && StringUtils.isNotBlank(measure.getName());
     }
 
-    public static List<Dim> getDimToCreateDimension(ModelDetail modelDetail) {
+    public static List<Dimension> getDimToCreateDimension(ModelDetail modelDetail) {
         if (CollectionUtils.isEmpty(modelDetail.getDimensions())) {
             return Lists.newArrayList();
         }
@@ -254,7 +255,7 @@ public class ModelConverter {
         List<DimensionReq> dimensionReqs = Lists.newArrayList();
         ModelDetail modelDetail =
                 JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class);
-        List<Dim> dims = getDimToCreateDimension(modelDetail);
+        List<Dimension> dims = getDimToCreateDimension(modelDetail);
         if (!CollectionUtils.isEmpty(dims)) {
             dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName()))
                     .map(dim -> convert(dim, modelDO)).collect(Collectors.toList());

headless/server/src/main/resources/mapper/StatMapper.xml

@@ -40,28 +40,4 @@
         <result column="query_opt_mode" property="queryOptMode"/>
     </resultMap>
 
-    <select id="getStatInfo"
-            resultType="com.tencent.supersonic.headless.api.pojo.QueryStat">
-        select *
-        from s2_query_stat_info
-        <where>
-            <if test="startTime != null">
-                and start_time >= #{startTime}String.valueOf(queryFilter.getValue())
-            </if>
-            <if test="modelId != null">
-                and model_id = #{modelId}
-            </if>
-            <if test="modelIds != null and modelIds.size() > 0">
-                and model_id in
-                <foreach item="id" collection="modelIds" open="(" separator="," close=")">
-                    #{id}
-                </foreach>
-            </if>
-            <if test="metric != null">
-                and metrics like concat('%',#{metric},'%')
-            </if>
-        </where>
-
-    </select>
-
 </mapper>

headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml

@@ -116,63 +116,6 @@
         </foreach>
     </update>
 
-    <select id="query" resultMap="ResultMapWithBLOBs">
-        select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
-        from s2_dimension t
-        left join (
-             select *
-             from s2_tag
-             where type = 'DIMENSION'
-        ) t1 on t.id = t1.item_id
-        where status != 3
-        <if test="key != null and key != ''">
-            and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
-        </if>
-        <if test="id != null">
-            and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="name != null and name != '' ">
-            and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="bizName != null and bizName != ''">
-            and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="sensitiveLevel != null">
-            and t.sensitive_level = #{sensitiveLevel}
-        </if>
-        <if test="status != null">
-            and t.status = #{status}
-        </if>
-        <if test="modelIds != null and modelIds.size >0">
-            and t.model_id in
-            <foreach collection="modelIds" index="index" item="model" open="(" close=")"
-                     separator=",">
-                #{model}
-            </foreach>
-        </if>
-        <if test="ids != null and ids.size >0">
-            and t.id in
-            <foreach collection="ids" index="index" item="id" open="(" close=")"
-                     separator=",">
-                #{id}
-            </foreach>
-        </if>
-        <if test="createdBy != null">
-            and t.created_by = #{createdBy}
-        </if>
-        <if test="isTag != null and isTag == 1">
-            and t1.id is not null
-        </if>
-        <if test="isTag != null and isTag == 0">
-            and t1.id is null
-        </if>
-    </select>
-
-
     <select id="queryDimensions" resultMap="ResultMapWithBLOBs">
         select *
         from s2_dimension

headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml

@@ -127,70 +127,6 @@
         </foreach>
     </update>
 
-    <select id="query" resultMap="ResultMapWithBLOBs">
-        select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
-        from s2_metric t
-        left join (
-           select *
-           from s2_tag
-           where type = 'METRIC'
-        ) t1 on t.id = t1.item_id
-        where t.status != 3
-        <if test="type != null and type != ''">
-            and t.type = #{type}
-        </if>
-        <if test="key != null and key != ''">
-            and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
-            t.classifications like CONCAT('%',#{key , jdbcType=VARCHAR},'%')  or
-            t.created_by like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
-        </if>
-        <if test="id != null">
-            and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="name != null and name != '' ">
-            and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="bizName != null and bizName != ''">
-            and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
-        </if>
-        <if test="sensitiveLevel != null">
-            and t.sensitive_level = #{sensitiveLevel}
-        </if>
-        <if test="status != null">
-            and t.status = #{status}
-        </if>
-        <if test="modelIds != null and modelIds.size >0">
-            and t.model_id in
-            <foreach collection="modelIds" index="index" item="model" open="(" close=")"
-                    separator=",">
-                #{model}
-            </foreach>
-        </if>
-        <if test="ids != null and ids.size >0">
-            and t.id in
-            <foreach collection="ids" index="index" item="id" open="(" close=")"
-                    separator=",">
-                #{id}
-            </foreach>
-        </if>
-        <if test="createdBy != null">
-            and t.created_by = #{createdBy}
-        </if>
-        <if test="isTag != null and isTag == 1">
-            and t1.id is not null
-        </if>
-        <if test="isTag != null and isTag == 0">
-            and t1.id is null
-        </if>
-        <if test="isPublish != null and isPublish == 1">
-           and (t.created_by = #{userName} or t.is_publish = 1)
-        </if>
-    </select>
-
     <select id="queryMetrics" resultMap="ResultMapWithBLOBs">
         select *
         from s2_metric

headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
-import com.tencent.supersonic.headless.api.pojo.Dim;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
 import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -95,11 +95,11 @@ class ModelServiceImplTest {
         List<Identify> identifiers = new ArrayList<>();
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -134,11 +134,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名_a", IdentifyType.primary.name(), "user_name_a"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date_a", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date_a", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page_a", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page_a", DimensionType.categorical, 0);
         dimension2.setExpr("page_a");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -169,11 +169,11 @@ class ModelServiceImplTest {
         List<Identify> identifiers = new ArrayList<>();
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -207,11 +207,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page", DimensionType.categorical, 0);
         dimension2.setExpr("page");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);
@@ -252,11 +252,11 @@ class ModelServiceImplTest {
         identifiers.add(new Identify("用户名_a", IdentifyType.primary.name(), "user_name_a"));
         modelDetail.setIdentifiers(identifiers);
 
-        List<Dim> dimensions = new ArrayList<>();
-        Dim dimension1 = new Dim("", "imp_date_a", DimensionType.partition_time, 0);
+        List<Dimension> dimensions = new ArrayList<>();
+        Dimension dimension1 = new Dimension("", "imp_date_a", DimensionType.partition_time, 0);
         dimension1.setTypeParams(new DimensionTimeTypeParams());
         dimensions.add(dimension1);
-        Dim dimension2 = new Dim("", "page_a", DimensionType.categorical, 0);
+        Dimension dimension2 = new Dimension("", "page_a", DimensionType.categorical, 0);
         dimension2.setExpr("page_a");
         dimensions.add(dimension2);
         modelDetail.setDimensions(dimensions);