Refactor translator module (#1932)

* [improvement][Chat] Support agent permission management #1143

* [improvement][chat]Iterate LLM prompts of parsing and correction.

* [improvement][headless]Clean code logic of headless core.

* (fix) (chat) 记忆管理更新不生效 (#1912)

* [improvement][headless-fe] Added null-check conditions to the data formatting function.

* [improvement][headless]Clean code logic of headless translator.

* [improvement][headless-fe] Added permissions management for agents.

* [improvement][headless-fe] Unified the assistant's permission settings interaction to match the system style.

* [improvement](Dict)Support returns dict task list of dimensions by page

* [improvement][headless-fe] Revised the interaction for semantic modeling routing and implemented the initial version of metric management switching.

* [improvement][launcher]Set system property `s2.test` in junit tests in order to facilitate conditional breakpoints.

* [improvement][headless] add validateAndQuery interface in SqlQueryApiController

* [improvement][launcher]Use API to get element ID avoiding hard-code.

* [improvement][launcher]Support DuckDB database and refactor translator code structure.

---------

Co-authored-by: lxwcodemonkey <jolunoluo@tencent.com>
Co-authored-by: tristanliu <tristanliu@tencent.com>
Co-authored-by: daikon12 <1059907724@qq.com>
Co-authored-by: lexluo09 <39718951+lexluo09@users.noreply.github.com>

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java

@@ -6,6 +6,7 @@ import javax.servlet.http.HttpServletResponse;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.util.StringUtil;
+import com.tencent.supersonic.headless.api.pojo.SqlEvaluation;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlsReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
@@ -102,4 +103,28 @@ public class SqlQueryApiController {
         querySqlReq.setSql(StringUtil.replaceBackticks(sql));
         return chatLayerService.validate(querySqlReq, user);
     }
+
+    @PostMapping("/validateAndQuery")
+    public Object validateAndQuery(@RequestBody QuerySqlsReq querySqlsReq,
+            HttpServletRequest request, HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        List<QuerySqlReq> convert = convert(querySqlsReq);
+        for (QuerySqlReq querySqlReq : convert) {
+            SqlEvaluation validate = chatLayerService.validate(querySqlReq, user);
+            if (!validate.getIsValidated()) {
+                throw new Exception(validate.getValidateMsg());
+            }
+        }
+        return queryBySqls(querySqlsReq, request, response);
+    }
+
+    private List<QuerySqlReq> convert(QuerySqlsReq querySqlsReq) {
+        return querySqlsReq.getSqls().stream().map(sql -> {
+            QuerySqlReq querySqlReq = new QuerySqlReq();
+            BeanUtils.copyProperties(querySqlsReq, querySqlReq);
+            querySqlReq.setSql(StringUtil.replaceBackticks(sql));
+            return querySqlReq;
+        }).collect(Collectors.toList());
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -6,24 +6,10 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
-import com.tencent.supersonic.headless.api.pojo.Dim;
-import com.tencent.supersonic.headless.api.pojo.MetaFilter;
-import com.tencent.supersonic.headless.api.pojo.QueryParam;
+import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
-import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
-import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
-import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp;
+import com.tencent.supersonic.headless.api.pojo.request.*;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
 import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
 import com.tencent.supersonic.headless.chat.knowledge.MapResult;
@@ -33,6 +19,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.core.cache.QueryCache;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
+import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
 import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
 import com.tencent.supersonic.headless.core.utils.ComponentFactory;
 import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
@@ -52,12 +40,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 @Service
@@ -307,30 +290,13 @@ public class S2SemanticLayerService implements SemanticLayerService {
         return queryStatement;
     }
 
-    private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) {
-        // If dataSetId or DataSetName is empty, parse dataSetId from the SQL
-        if (querySqlReq.needGetDataSetId()) {
-            Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
-            querySqlReq.setDataSetId(dataSetId);
-        }
-
-        QueryStatement queryStatement = buildStructQueryStatement(querySqlReq);
-        queryStatement.setIsS2SQL(true);
-        queryStatement.setSql(querySqlReq.getSql());
-        return queryStatement;
-    }
-
-    private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) {
+    private QueryStatement buildQueryStatement(SemanticQueryReq queryReq) {
         SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
         schemaFilterReq.setDataSetId(queryReq.getDataSetId());
         schemaFilterReq.setModelIds(queryReq.getModelIds());
         SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(schemaFilterReq);
 
         QueryStatement queryStatement = new QueryStatement();
-        QueryParam queryParam = new QueryParam();
-        BeanUtils.copyProperties(queryReq, queryParam);
-        queryStatement.setQueryParam(queryParam);
-        queryStatement.setModelIds(queryReq.getModelIds());
         queryStatement.setEnableOptimize(queryUtils.enableOptimize());
         queryStatement.setDataSetId(queryReq.getDataSetId());
         queryStatement.setSemanticSchemaResp(semanticSchemaResp);
@@ -338,6 +304,31 @@ public class S2SemanticLayerService implements SemanticLayerService {
         return queryStatement;
     }
 
+    private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) {
+        QueryStatement queryStatement = buildQueryStatement(querySqlReq);
+        queryStatement.setIsS2SQL(true);
+
+        SqlQueryParam sqlQueryParam = new SqlQueryParam();
+        sqlQueryParam.setSql(querySqlReq.getSql());
+        queryStatement.setSqlQueryParam(sqlQueryParam);
+
+        // If dataSetId or DataSetName is empty, parse dataSetId from the SQL
+        if (querySqlReq.needGetDataSetId()) {
+            Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
+            querySqlReq.setDataSetId(dataSetId);
+        }
+        return queryStatement;
+    }
+
+    private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) {
+        QueryStatement queryStatement = buildQueryStatement(queryReq);
+        StructQueryParam structQueryParam = new StructQueryParam();
+        BeanUtils.copyProperties(queryReq, structQueryParam);
+        queryStatement.setStructQueryParam(structQueryParam);
+        queryStatement.setIsS2SQL(false);
+        return queryStatement;
+    }
+
     private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq) {
         List<QueryStatement> queryStatements = new ArrayList<>();
         for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -33,7 +33,6 @@ public class ModelYamlManager {
         ModelDetail modelDetail = modelResp.getModelDetail();
         DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
         SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
-        addInterCntMetric(modelResp.getBizName(), modelDetail);
         DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
         dataModelYamlTpl.setType(databaseResp.getType());
         BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -2,33 +2,12 @@ package com.tencent.supersonic.headless.server.manager;
 
 import com.tencent.supersonic.common.pojo.ModelRela;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.headless.api.pojo.Field;
-import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.TagResp;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization.TimePartType;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams;
-import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
-import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
-import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
+import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
+import com.tencent.supersonic.headless.core.translator.parser.s2sql.*;
+import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType;
+import com.tencent.supersonic.headless.server.pojo.yaml.*;
 import com.tencent.supersonic.headless.server.service.SchemaService;
 import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
 import lombok.extern.slf4j.Slf4j;
@@ -36,15 +15,8 @@ import org.apache.commons.lang3.tuple.Triple;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -90,82 +62,6 @@ public class SemanticSchemaManager {
         return ontology;
     }
 
-    public Ontology getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
-        if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
-            throw new Exception("semanticSchemaResp tag is empty");
-        }
-        Ontology ontology = buildOntology(semanticSchemaResp);
-        // Map<String, List<Dimension>> dimensions = new HashMap<>();
-        Map<Long, List<TagResp>> tagMap = new HashMap<>();
-        for (TagResp tagResp : semanticSchemaResp.getTags()) {
-            if (!tagMap.containsKey(tagResp.getModelId())) {
-                tagMap.put(tagResp.getModelId(), new ArrayList<>());
-            }
-            tagMap.get(tagResp.getModelId()).add(tagResp);
-        }
-        if (Objects.nonNull(ontology.getDataModelMap()) && !ontology.getDataModelMap().isEmpty()) {
-            for (Map.Entry<String, DataModel> entry : ontology.getDataModelMap().entrySet()) {
-                List<Dimension> modelDimensions = new ArrayList<>();
-                if (!ontology.getDimensionMap().containsKey(entry.getKey())) {
-                    ontology.getDimensionMap().put(entry.getKey(), modelDimensions);
-                } else {
-                    modelDimensions = ontology.getDimensionMap().get(entry.getKey());
-                }
-                if (tagMap.containsKey(entry.getValue().getId())) {
-                    for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
-                        addTagModel(tagResp, modelDimensions, ontology.getMetrics());
-                    }
-                }
-            }
-        }
-
-        return ontology;
-    }
-
-    private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions,
-            List<Metric> modelMetrics) throws Exception {
-        TagDefineType tagDefineType = TagDefineType.valueOf(tagResp.getTagDefineType());
-        switch (tagDefineType) {
-            case FIELD:
-            case DIMENSION:
-                if (TagDefineType.DIMENSION.equals(tagResp.getTagDefineType())) {
-                    Optional<Dimension> modelDimension = modelDimensions.stream()
-                            // .filter(d -> d.getBizName().equals(tagResp.getExpr()))
-                            .findFirst();
-                    if (modelDimension.isPresent()) {
-                        modelDimension.get().setName(tagResp.getBizName());
-                        return;
-                    }
-                }
-                Dimension dimension = Dimension.builder().build();
-                dimension.setType("");
-                // dimension.setExpr(tagResp.getExpr());
-                dimension.setName(tagResp.getBizName());
-                dimension.setOwners("");
-                dimension.setBizName(tagResp.getBizName());
-                if (Objects.isNull(dimension.getDataType())) {
-                    dimension.setDataType(DataType.UNKNOWN);
-                }
-
-                DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
-                dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
-                modelDimensions.add(dimension);
-                return;
-            case METRIC:
-                Optional<Metric> modelMetric = modelMetrics.stream()
-                        // .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
-                        .findFirst();
-                if (modelMetric.isPresent()) {
-                    modelMetric.get().setName(tagResp.getBizName());
-                } else {
-                    throw new Exception(
-                            String.format("tag [{}] cant find the metric", tagResp.getBizName()));
-                }
-                return;
-            default:
-        }
-    }
-
     public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
         return getMetricsByMetricYamlTpl(t);
     }
@@ -184,16 +80,6 @@ public class SemanticSchemaManager {
         if (Objects.nonNull(d.getModelSourceTypeEnum())) {
             dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
         }
-        if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
-            Set<String> measures = dataModel.getMeasures().stream().map(mm -> mm.getName())
-                    .collect(Collectors.toSet());
-            for (Field f : d.getFields()) {
-                if (!measures.contains(f.getFieldName())) {
-                    dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName())
-                            .name(f.getFieldName()).agg("").build());
-                }
-            }
-        }
         return dataModel;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DictRepository.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.persistence.repository;
 
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
+import com.tencent.supersonic.headless.api.pojo.request.ValueTaskQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
@@ -26,4 +27,6 @@ public interface DictRepository {
     DictTaskDO queryDictTaskById(Long id);
 
     DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq);
+
+    List<DictTaskDO> queryAllDictTask(ValueTaskQueryReq taskQueryReq);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
+import com.tencent.supersonic.headless.api.pojo.request.ValueTaskQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
 import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -14,11 +15,14 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DictTaskMapper;
 import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
+import com.xkzhangsan.time.utils.CollectionUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.codehaus.plexus.util.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
@@ -100,6 +104,23 @@ public class DictRepositoryImpl implements DictRepository {
         return taskResp;
     }
 
+    @Override
+    public List<DictTaskDO> queryAllDictTask(ValueTaskQueryReq taskQueryReq) {
+        QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
+        if (Objects.nonNull(taskQueryReq.getItemId())) {
+            wrapper.lambda().eq(DictTaskDO::getItemId, taskQueryReq.getItemId());
+        }
+        if (CollectionUtil.isNotEmpty(taskQueryReq.getTaskStatusList())) {
+            wrapper.lambda().in(DictTaskDO::getStatus, taskQueryReq.getTaskStatusList());
+        }
+        if (StringUtils.isNotEmpty(taskQueryReq.getKey())) {
+            String key = taskQueryReq.getKey();
+            wrapper.lambda().and(qw -> qw.like(DictTaskDO::getName, key).or()
+                    .like(DictTaskDO::getDescription, key).or().like(DictTaskDO::getConfig, key));
+        }
+        return dictTaskMapper.selectList(wrapper);
+    }
+
     @Override
     public Long addDictConf(DictConfDO dictConfDO) {
         dictConfMapper.insert(dictConfDO);

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java

@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.ValueTaskQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
@@ -132,6 +133,18 @@ public class KnowledgeController {
         return taskService.queryLatestDictTask(taskReq, user);
     }
 
+    /**
+     * queryDictTask-分页返回维度的字典任务列表
+     *
+     * @param taskQueryReq
+     */
+    @PostMapping("/task/search/page")
+    public PageInfo<DictTaskResp> queryDictTask(@RequestBody ValueTaskQueryReq taskQueryReq,
+            HttpServletRequest request, HttpServletResponse response) {
+        User user = UserHolder.findUser(request, response);
+        return taskService.queryDictTask(taskQueryReq, user);
+    }
+
     @GetMapping("/embedding/reload")
     public Object reloadEmbedding() {
         metaEmbeddingTask.reloadMetaEmbedding();

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java

@@ -4,6 +4,7 @@ import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.ValueTaskQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
 
@@ -17,6 +18,8 @@ public interface DictTaskService {
 
     DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user);
 
+    PageInfo<DictTaskResp> queryDictTask(ValueTaskQueryReq taskQueryReq, User user);
+
     PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user);
 
     String queryDictFilePath(DictValueReq dictValueReq, User user);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java

@@ -9,15 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.MetaFilter;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
-import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
-import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
-import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
-import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
-import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.*;
 import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
 import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
@@ -64,5 +56,4 @@ public interface SchemaService {
 
     ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
 
-    DatabaseResp getDatabase(Long id);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java

@@ -1,15 +1,18 @@
 package com.tencent.supersonic.headless.server.service.impl;
 
+import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
 import com.tencent.supersonic.common.util.BeanMapper;
+import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.ValueTaskQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
@@ -116,14 +119,17 @@ public class DictTaskServiceImpl implements DictTaskService {
         fileHandler.writeFile(data, fileName, false);
 
         // 3.Change in-memory dictionary data in real time
+        String status = TaskStatusEnum.SUCCESS.getStatus();
         try {
             dictWordService.loadDictWord();
-
-            dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
-            dictRepository.editDictTask(dictTaskDO);
         } catch (Exception e) {
             log.error("reloadCustomDictionary error", e);
+            status = TaskStatusEnum.ERROR.getStatus();
+            dictTaskDO.setDescription(e.toString());
         }
+        dictTaskDO.setStatus(status);
+        dictTaskDO.setElapsedMs(DateUtils.calculateDiffMs(dictTaskDO.getCreatedAt()));
+        dictRepository.editDictTask(dictTaskDO);
     }
 
     @Override
@@ -164,6 +170,17 @@ public class DictTaskServiceImpl implements DictTaskService {
         return dictRepository.queryLatestDictTask(taskReq);
     }
 
+    @Override
+    public PageInfo<DictTaskResp> queryDictTask(ValueTaskQueryReq taskQueryReq, User user) {
+        PageInfo<DictTaskDO> dictTaskDOPageInfo =
+                PageHelper.startPage(taskQueryReq.getCurrent(), taskQueryReq.getPageSize())
+                        .doSelectPageInfo(() -> dictRepository.queryAllDictTask(taskQueryReq));
+        PageInfo<DictTaskResp> dictTaskRespPageInfo = new PageInfo<>();
+        BeanMapper.mapper(dictTaskDOPageInfo, dictTaskRespPageInfo);
+        dictTaskRespPageInfo.setList(dictConverter.taskDO2Resp(dictTaskDOPageInfo.getList()));
+        return dictTaskRespPageInfo;
+    }
+
     @Override
     public PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user) {
         // todo 优化读取内存结构

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java

@@ -536,8 +536,4 @@ public class SchemaServiceImpl implements SchemaService {
         return modelService.getItemDate(dimension, metric);
     }
 
-    @Override
-    public DatabaseResp getDatabase(Long id) {
-        return databaseService.getDatabase(id);
-    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.server.utils;
 
 import com.alibaba.fastjson.JSONObject;
+import com.tencent.supersonic.common.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
@@ -16,6 +17,7 @@ public class DatabaseConverter {
     public static Database convert(DatabaseResp databaseResp) {
         Database database = new Database();
         BeanUtils.copyProperties(databaseResp, database);
+        database.setType(EngineType.fromString(databaseResp.getType().toUpperCase()));
         return database;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.xkzhangsan.time.utils.CollectionUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
@@ -533,4 +534,12 @@ public class DictUtils {
         resp.setConfig(JsonUtil.toObject(dictTaskDO.getConfig(), ItemValueConfig.class));
         return resp;
     }
+
+    public List<DictTaskResp> taskDO2Resp(List<DictTaskDO> dictTaskDOList) {
+        List<DictTaskResp> dictTaskRespList = new ArrayList<>();
+        if (CollectionUtil.isNotEmpty(dictTaskDOList)) {
+            dictTaskDOList.stream().forEach(taskDO -> dictTaskRespList.add(taskDO2Resp(taskDO)));
+        }
+        return dictTaskRespList;
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java

@@ -67,7 +67,8 @@ public class MetricDrillDownChecker {
             List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
             if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
                 DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);
-                if (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime()) {
+                if (Objects.isNull(dimSchemaResp)
+                        || (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime())) {
                     continue;
                 }
                 String errMsg =

headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java

@@ -2,31 +2,25 @@ package com.tencent.supersonic.headless.server.calcite;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
 import com.tencent.supersonic.common.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
-import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
-import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
-import com.tencent.supersonic.headless.core.translator.calcite.sql.SqlBuilder;
+import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
+import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder;
+import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam;
 import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
-import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
-import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
+import com.tencent.supersonic.headless.server.pojo.yaml.*;
 import lombok.extern.slf4j.Slf4j;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 
 @Slf4j
 class HeadlessParserServiceTest {
 
     public static SqlParserResp parser(S2CalciteSchema semanticSchema,
-            MetricQueryParam metricQueryParam, boolean isAgg) {
+            OntologyQueryParam ontologyQueryParam, boolean isAgg) {
         SqlParserResp sqlParser = new SqlParserResp();
         try {
             if (semanticSchema == null) {
@@ -35,14 +29,14 @@ class HeadlessParserServiceTest {
             }
             SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
             QueryStatement queryStatement = new QueryStatement();
-            queryStatement.setMetricQueryParam(metricQueryParam);
-            aggBuilder.build(queryStatement, AggOption.getAggregation(!isAgg));
-            EngineType engineType =
-                    EngineType.fromString(semanticSchema.getOntology().getDatabase().getType());
+            queryStatement.setOntologyQueryParam(ontologyQueryParam);
+            String sql = aggBuilder.buildOntologySql(queryStatement);
+            queryStatement.setSql(sql);
+            EngineType engineType = semanticSchema.getOntology().getDatabase().getType();
             sqlParser.setSql(aggBuilder.getSql(engineType));
         } catch (Exception e) {
             sqlParser.setErrMsg(e.getMessage());
-            log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
+            log.error("parser error metricQueryReq[{}] error [{}]", ontologyQueryParam, e);
         }
         return sqlParser;
     }
@@ -161,9 +155,9 @@ class HeadlessParserServiceTest {
 
         // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
 
-        MetricQueryParam metricCommand = new MetricQueryParam();
-        metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
-        metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
+        OntologyQueryParam metricCommand = new OntologyQueryParam();
+        metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
+        metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
         metricCommand.setWhere(
                 "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
         metricCommand.setLimit(1000L);
@@ -174,10 +168,10 @@ class HeadlessParserServiceTest {
 
         addDepartment(semanticSchema);
 
-        MetricQueryParam metricCommand2 = new MetricQueryParam();
-        metricCommand2.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date",
+        OntologyQueryParam metricCommand2 = new OntologyQueryParam();
+        metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
                 "user_name__department", "user_name", "user_name__page")));
-        metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));
+        metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
         metricCommand2.setWhere(
                 "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
         metricCommand2.setLimit(1000L);

headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java

@@ -68,8 +68,5 @@ class QueryNLReqBuilderTest {
         DateModeUtils dateModeUtils = new DateModeUtils();
         mockContextUtils.when(() -> ContextUtils.getBean(DateModeUtils.class))
                 .thenReturn(dateModeUtils);
-        dateModeUtils.setSysDateCol("sys_imp_date");
-        dateModeUtils.setSysDateWeekCol("sys_imp_week");
-        dateModeUtils.setSysDateMonthCol("sys_imp_month");
     }
 }