(improvement)(chat) Fix agent filter rule query (#777)

Co-authored-by: jolunoluo

.github/workflows/mac-ci.yml

@@ -0,0 +1,35 @@
+name: supersonic mac CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: macos-latest  # Specify a macOS runner
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/Library/Caches/Maven  # macOS Maven cache path
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Test with Maven
+        run: mvn test

.github/workflows/ubuntu-ci.yml

@@ -1,4 +1,4 @@
-name: supersonic CI
+name: supersonic ubuntu CI
 
 on:
   push:

.github/workflows/windows-ci.yml

@@ -0,0 +1,35 @@
+name: supersonic windows CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: windows-latest  # Specify a Windows runner
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~\.m2  # Windows uses a backslash for paths
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Test with Maven
+        run: mvn test

README.md

@@ -4,7 +4,7 @@
 
 # SuperSonic (超音数)
 
-**SuperSonic is the next-generation LLM-powered data analytics platform that integrates ChatBI and HeadlessBI**. SuperSonic provides a chat interface that empowers users to query data using natural language and visualize the results with suitable charts. To enable such experience, the only thing necessary is to build logical semantic models (definition of entities/metrics/dimensions/tags, along with their meaning, context and relationships) on top of physical data models, and **no data modification or copying** is required. Meanwhile, SuperSonic is designed to be **highly extensible**, allowing custom functionalities to be added and configured with Java SPI.
+**SuperSonic is the next-generation LLM-powered data analytics platform that integrates ChatBI and HeadlessBI**. SuperSonic provides a chat interface that empowers users to query data using natural language and visualize the results with suitable charts. To enable such experience, the only thing necessary is to build logical semantic models (definition of entities/metrics/dimensions/tags, along with their meaning, context and relationships) with semantic layer, and **no data modification or copying** is required. Meanwhile, SuperSonic is designed to be **highly extensible**, allowing custom functionalities to be added and configured with Java SPI.
 
 <img src="./docs/images/supersonic_demo.gif" height="100%" width="100%" align="center"/>
 
@@ -13,7 +13,8 @@
 The emergence of Large Language Model (LLM) like ChatGPT is reshaping the way information is retrieved. In the field of data analytics, both academia and industry are primarily focused on leveraging LLM to convert natural language into SQL (so called Text2SQL or NL2SQL). While some approaches exhibit promising results, their **reliability** and **efficiency** are insufficient for real-world applications.  
 
 From our perspective, the key to filling the real-world gap lies in three aspects: 
-1. Integrate ChatBI with HeadlessBI encapsulating underlying data context (joins, keys, formulas, etc) to **reduce complexity**.
+1. Integrate ChatBI with HeadlessBI encapsulating underlying data context (joins, keys, formulas, etc) to **reduce complexity**. 
+   <img src="./docs/images/supersonic_ideas.png" height="65%" width="65%" align="center"/>
 2. Augment the LLM with schema mappers(as a kind of preprocessor) and semantic correctors(as a kind of postprocessor) to **mitigate hallucination**.
 3. Utilize rule-based schema parsers when necessary to **improve efficiency**(in terms of latency and cost).
 

README_CN.md

@@ -10,6 +10,8 @@
 
 在我们看来，为了在实际场景发挥价值，有三个关键点：
 1. 融合HeadlessBI，通过统一语义层封装底层数据细节（关联、键值、公式等），降低SQL生成的**复杂度**。
+
+   <img src="./docs/images/supersonic_ideas.png" height="65%" width="65%" align="center"/>
 2. 通过一前一后的模式映射器和语义修正器，来缓解LLM常见的**幻觉**现象。
 3. 设计启发式的规则，在一些特定场景提升语义解析的**效率**。
 

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java

@@ -7,7 +7,7 @@ import lombok.Data;
 public class QueryReq {
     private String queryText;
     private Integer chatId;
-    private Long modelId;
+    private Long viewId;
     private User user;
     private QueryFilters queryFilters;
     private boolean saveAnswer = true;

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java

@@ -140,4 +140,19 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
         return semanticSchema.getMetrics(viewId);
     }
 
+    protected Set<String> getDimensions(Long viewId, SemanticSchema semanticSchema) {
+        Set<String> dimensions = semanticSchema.getDimensions(viewId).stream()
+                .flatMap(
+                        schemaElement -> {
+                            Set<String> elements = new HashSet<>();
+                            elements.add(schemaElement.getName());
+                            if (!CollectionUtils.isEmpty(schemaElement.getAlias())) {
+                                elements.addAll(schemaElement.getAlias());
+                            }
+                            return elements.stream();
+                        }
+                ).collect(Collectors.toSet());
+        dimensions.add(TimeDimensionEnum.DAY.getChName());
+        return dimensions;
+    }
 }

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GrammarCorrector.java

@@ -0,0 +1,31 @@
+package com.tencent.supersonic.chat.core.corrector;
+
+import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.chat.core.pojo.QueryContext;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Correcting SQL syntax, primarily including fixes to select, where, groupBy, and Having clauses
+ */
+@Slf4j
+public class GrammarCorrector extends BaseSemanticCorrector {
+
+    private List<BaseSemanticCorrector> correctors;
+
+    public GrammarCorrector() {
+        correctors = new ArrayList<>();
+        correctors.add(new SelectCorrector());
+        correctors.add(new WhereCorrector());
+        correctors.add(new GroupByCorrector());
+        correctors.add(new HavingCorrector());
+    }
+
+    @Override
+    public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
+        for (BaseSemanticCorrector corrector : correctors) {
+            corrector.correct(queryContext, semanticParseInfo);
+        }
+    }
+}

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java

@@ -14,14 +14,12 @@ import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.ViewService;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.util.CollectionUtils;
-
-import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
 
 /**
  * Perform SQL corrections on the "Group by" section in S2SQL.
@@ -82,22 +80,6 @@ public class GroupByCorrector extends BaseSemanticCorrector {
         return true;
     }
 
-    private Set<String> getDimensions(Long viewId, SemanticSchema semanticSchema) {
-        Set<String> dimensions = semanticSchema.getDimensions(viewId).stream()
-                .flatMap(
-                        schemaElement -> {
-                            Set<String> elements = new HashSet<>();
-                            elements.add(schemaElement.getName());
-                            if (!CollectionUtils.isEmpty(schemaElement.getAlias())) {
-                                elements.addAll(schemaElement.getAlias());
-                            }
-                            return elements.stream();
-                        }
-                ).collect(Collectors.toSet());
-        dimensions.add(TimeDimensionEnum.DAY.getChName());
-        return dimensions;
-    }
-
     private void addGroupByFields(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
         Long viewId = semanticParseInfo.getViewId();
         //add dimension group by

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java

@@ -1,22 +1,30 @@
 package com.tencent.supersonic.chat.core.corrector;
 
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
 import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue;
 import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum;
+import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
+import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.util.CollectionUtils;
-
+import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.CollectionUtils;
 
 /**
  * Perform schema corrections on the Schema information in S2SQL.
@@ -27,6 +35,8 @@ public class SchemaCorrector extends BaseSemanticCorrector {
     @Override
     public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
 
+        removeFilterIfNotInLinkingValue(queryContext, semanticParseInfo);
+
         correctAggFunction(semanticParseInfo);
 
         replaceAlias(semanticParseInfo);
@@ -105,4 +115,35 @@ public class SchemaCorrector extends BaseSemanticCorrector {
         String sql = SqlReplaceHelper.replaceValue(sqlInfo.getCorrectS2SQL(), filedNameToValueMap, false);
         sqlInfo.setCorrectS2SQL(sql);
     }
+
+    public void removeFilterIfNotInLinkingValue(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
+        SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
+        String correctS2SQL = sqlInfo.getCorrectS2SQL();
+        List<FieldExpression> whereExpressionList = SqlSelectHelper.getWhereExpressions(correctS2SQL);
+        if (CollectionUtils.isEmpty(whereExpressionList)) {
+            return;
+        }
+        List<ElementValue> linkingValues = getLinkingValues(semanticParseInfo);
+        SemanticSchema semanticSchema = queryContext.getSemanticSchema();
+        Set<String> dimensions = getDimensions(semanticParseInfo.getViewId(), semanticSchema);
+
+        if (CollectionUtils.isEmpty(linkingValues)) {
+            linkingValues = new ArrayList<>();
+        }
+        Set<String> linkingFieldNames = linkingValues.stream().map(linking -> linking.getFieldName())
+                .collect(Collectors.toSet());
+
+        Set<String> removeFieldNames = whereExpressionList.stream()
+                .filter(fieldExpression -> StringUtils.isBlank(fieldExpression.getFunction()))
+                .filter(fieldExpression -> !TimeDimensionEnum.containsTimeDimension(fieldExpression.getFieldName()))
+                .filter(fieldExpression -> FilterOperatorEnum.EQUALS.getValue().equals(fieldExpression.getOperator()))
+                .filter(fieldExpression -> dimensions.contains(fieldExpression.getFieldName()))
+                .filter(fieldExpression -> !DateUtils.isAnyDateString(fieldExpression.getFieldValue().toString()))
+                .filter(fieldExpression -> !linkingFieldNames.contains(fieldExpression.getFieldName()))
+                .map(fieldExpression -> fieldExpression.getFieldName()).collect(Collectors.toSet());
+
+        String sql = SqlRemoveHelper.removeWhereCondition(correctS2SQL, removeFieldNames);
+        sqlInfo.setCorrectS2SQL(sql);
+    }
+
 }

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java

@@ -23,20 +23,22 @@ public class AgentCheckParser implements SemanticParser {
     @Override
     public void parse(QueryContext queryContext, ChatContext chatContext) {
         List<SemanticQuery> queries = queryContext.getCandidateQueries();
-        agentCanSupport(queryContext, queries);
+        log.info("query size before agent filter:{}", queryContext.getCandidateQueries().size());
+        filterQueries(queryContext, queries);
+        log.info("query size after agent filter: {}", queryContext.getCandidateQueries().size());
     }
 
-    private void agentCanSupport(QueryContext queryContext, List<SemanticQuery> queries) {
+    private void filterQueries(QueryContext queryContext, List<SemanticQuery> queries) {
         Agent agent = queryContext.getAgent();
         if (agent == null) {
             return;
         }
         List<RuleParserTool> queryTools = getRuleTools(agent);
         if (CollectionUtils.isEmpty(queryTools)) {
-            queries.clear();
+            queryContext.setCandidateQueries(Lists.newArrayList());
             return;
         }
-        log.info("queries resolved:{} {}", agent.getName(),
+        log.info("agent name :{}, queries resolved: {}", agent.getName(),
                 queries.stream().map(SemanticQuery::getQueryMode).collect(Collectors.toList()));
         queries.removeIf(query -> {
             for (RuleParserTool tool : queryTools) {
@@ -46,10 +48,14 @@ public class AgentCheckParser implements SemanticParser {
                 }
                 if (CollectionUtils.isNotEmpty(tool.getQueryTypes())) {
                     if (QueryManager.isTagQuery(query.getQueryMode())) {
-                        return !tool.getQueryTypes().contains(QueryType.TAG.name());
+                        if (!tool.getQueryTypes().contains(QueryType.TAG.name())) {
+                            return true;
+                        }
                     }
                     if (QueryManager.isMetricQuery(query.getQueryMode())) {
-                        return !tool.getQueryTypes().contains(QueryType.METRIC.name());
+                        if (!tool.getQueryTypes().contains(QueryType.METRIC.name())) {
+                            return true;
+                        }
                     }
                 }
                 if (CollectionUtils.isEmpty(tool.getViewIds())) {
@@ -62,7 +68,8 @@ public class AgentCheckParser implements SemanticParser {
             }
             return true;
         });
-        log.info("rule queries witch can be supported by agent :{} {}", agent.getName(),
+        queryContext.setCandidateQueries(queries);
+        log.info("agent name :{}, rule queries witch can be supported by agent :{}", agent.getName(),
                 queries.stream().map(SemanticQuery::getQueryMode).collect(Collectors.toList()));
     }
 

chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java

@@ -0,0 +1,143 @@
+package com.tencent.supersonic.chat.core.corrector;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
+import com.tencent.supersonic.chat.api.pojo.ViewSchema;
+import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
+import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult;
+import com.tencent.supersonic.chat.core.pojo.QueryContext;
+import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.headless.api.pojo.QueryConfig;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.junit.jupiter.api.Test;
+
+class SchemaCorrectorTest {
+
+    private String json = "{\n"
+            + "          \"viewId\":  1,\n"
+            + "          \"llmReq\":  {\n"
+            + "                    \"queryText\":  \"xxx2024年播放量最高的十首歌\",\n"
+            + "                    \"filterCondition\":  {\n"
+            + "                              \"tableName\":  null\n"
+            + "                    },\n"
+            + "                    \"schema\":  {\n"
+            + "                              \"domainName\":  \"歌曲\",\n"
+            + "                              \"viewName\":  \"歌曲\",\n"
+            + "                              \"fieldNameList\":  [\n"
+            + "                                        \"商务组\",\n"
+            + "                                        \"歌曲名\",\n"
+            + "                                        \"播放量\",\n"
+            + "                                        \"播放份额\",\n"
+            + "                                        \"数据日期\"\n"
+            + "                              ]\n"
+            + "                    },\n"
+            + "                    \"linking\":  [\n"
+            + "\n"
+            + "                    ],\n"
+            + "                    \"currentDate\":  \"2024-02-24\",\n"
+            + "                    \"priorExts\":  \"播放份额是小数; \",\n"
+            + "                    \"sqlGenerationMode\":  \"2_pass_auto_cot\"\n"
+            + "          },\n"
+            + "          \"request\":  null,\n"
+            + "          \"commonAgentTool\":  {\n"
+            + "                    \"id\":  \"y3LqVSRL\",\n"
+            + "                    \"name\":  \"大模型语义解析\",\n"
+            + "                    \"type\":  \"NL2SQL_LLM\",\n"
+            + "                    \"viewIds\":  [\n"
+            + "                              1\n"
+            + "                    ]\n"
+            + "          },\n"
+            + "          \"linkingValues\":  [\n"
+            + "\n"
+            + "          ]\n"
+            + "}";
+
+    @Test
+    void doCorrect() throws JsonProcessingException {
+        Long viewId = 1L;
+        QueryContext queryContext = buildQueryContext(viewId);
+        ObjectMapper objectMapper = new ObjectMapper();
+        ParseResult parseResult = objectMapper.readValue(json, ParseResult.class);
+
+
+        String sql = "select  歌曲名 from 歌曲 where 发行日期 >= '2024-01-01' "
+                + "and 商务组 = 'xxx' order by 播放量 desc  limit 10";
+        SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
+        SqlInfo sqlInfo = new SqlInfo();
+        sqlInfo.setS2SQL(sql);
+        sqlInfo.setCorrectS2SQL(sql);
+        semanticParseInfo.setSqlInfo(sqlInfo);
+
+        SchemaElement schemaElement = new SchemaElement();
+        schemaElement.setView(viewId);
+        semanticParseInfo.setView(schemaElement);
+
+
+        semanticParseInfo.getProperties().put(Constants.CONTEXT, parseResult);
+
+        SchemaCorrector schemaCorrector = new SchemaCorrector();
+        schemaCorrector.removeFilterIfNotInLinkingValue(queryContext, semanticParseInfo);
+
+        assertEquals("SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' "
+                + "ORDER BY 播放量 DESC LIMIT 10", semanticParseInfo.getSqlInfo().getCorrectS2SQL());
+
+        parseResult = objectMapper.readValue(json, ParseResult.class);
+
+        List<ElementValue> linkingValues = new ArrayList<>();
+        ElementValue elementValue = new ElementValue();
+        elementValue.setFieldName("商务组");
+        elementValue.setFieldValue("xxx");
+        linkingValues.add(elementValue);
+        parseResult.setLinkingValues(linkingValues);
+        semanticParseInfo.getProperties().put(Constants.CONTEXT, parseResult);
+
+        semanticParseInfo.getSqlInfo().setCorrectS2SQL(sql);
+        semanticParseInfo.getSqlInfo().setS2SQL(sql);
+        schemaCorrector.removeFilterIfNotInLinkingValue(queryContext, semanticParseInfo);
+        assertEquals("SELECT 歌曲名 FROM 歌曲 WHERE 发行日期 >= '2024-01-01' "
+                + "AND 商务组 = 'xxx' ORDER BY 播放量 DESC LIMIT 10", semanticParseInfo.getSqlInfo().getCorrectS2SQL());
+
+    }
+
+    private QueryContext buildQueryContext(Long viewId) {
+        QueryContext queryContext = new QueryContext();
+        List<ViewSchema> viewSchemaList = new ArrayList<>();
+        ViewSchema viewSchema = new ViewSchema();
+        QueryConfig queryConfig = new QueryConfig();
+        viewSchema.setQueryConfig(queryConfig);
+        SchemaElement schemaElement = new SchemaElement();
+        schemaElement.setView(viewId);
+        viewSchema.setView(schemaElement);
+        Set<SchemaElement> dimensions = new HashSet<>();
+        SchemaElement element1 = new SchemaElement();
+        element1.setView(1L);
+        element1.setName("歌曲名");
+        dimensions.add(element1);
+
+        SchemaElement element2 = new SchemaElement();
+        element2.setView(1L);
+        element2.setName("商务组");
+        dimensions.add(element2);
+
+        SchemaElement element3 = new SchemaElement();
+        element3.setView(1L);
+        element3.setName("发行日期");
+        dimensions.add(element3);
+
+        viewSchema.setDimensions(dimensions);
+        viewSchemaList.add(viewSchema);
+
+        SemanticSchema semanticSchema = new SemanticSchema(viewSchemaList);
+        queryContext.setSemanticSchema(semanticSchema);
+        return queryContext;
+    }
+}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java

@@ -97,7 +97,7 @@ public class SearchServiceImpl implements SearchService {
         List<S2Term> originals = knowledgeService.getTerms(queryText);
         log.info("hanlp parse result: {}", originals);
         MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
-        Set<Long> detectViewIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId));
+        Set<Long> detectViewIds = mapperHelper.getViewIds(queryReq.getViewId(), agentService.getAgent(agentId));
 
         QueryContext queryContext = new QueryContext();
         BeanUtils.copyProperties(queryReq, queryContext);
@@ -123,7 +123,7 @@ public class SearchServiceImpl implements SearchService {
         Set<SearchResult> searchResults = new LinkedHashSet();
         ViewInfoStat modelStat = NatureHelper.getViewStat(originals);
 
-        List<Long> possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getModelId());
+        List<Long> possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getViewId());
 
         // 5.1 priority dimension metric
         boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleModels), modelToName,

common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java

@@ -19,8 +19,6 @@ public class Aggregator {
 
     private List<String> args;
 
-    private String alias;
-
     public Aggregator() {
     }
 

common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java

@@ -11,6 +11,7 @@ public class Constants {
     public static final String AT_SYMBOL = "@";
     public static final String DOT = ".";
     public static String SPACE = " ";
+    public static String POUND = "#";
     public static final String COLON = ":";
     public static final String MINUS = "-";
     public static final String UNDERLINE = "_";

common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java

@@ -2,6 +2,8 @@ package com.tencent.supersonic.common.pojo.enums;
 
 public enum TaskStatusEnum {
 
+    INITIAL("initial", -2),
+
     ERROR("error", -1),
 
     PENDING("pending", 0),
@@ -10,7 +12,7 @@ public enum TaskStatusEnum {
 
     SUCCESS("success", 2),
 
-    UNKNOWN("UNKNOWN", 3);
+    UNKNOWN("unknown", 3);
 
     private String status;
     private Integer code;

common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java

@@ -1,19 +1,21 @@
 package com.tencent.supersonic.common.util;
 
+import com.tencent.supersonic.common.pojo.Constants;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoField;
 import java.time.temporal.TemporalAdjuster;
 import java.time.temporal.TemporalAdjusters;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 import java.util.Objects;
-import com.tencent.supersonic.common.pojo.Constants;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -166,4 +168,27 @@ public class DateUtils {
         return datesInRange;
     }
 
+    public static boolean isAnyDateString(String value) {
+        List<String> formats = Arrays.asList("yyyy-MM-dd", "yyyy-MM", "yyyy/MM/dd");
+        return isAnyDateString(value, formats);
+    }
+
+    public static boolean isAnyDateString(String value, List<String> formats) {
+        for (String format : formats) {
+            if (isDateString(value, format)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean isDateString(String value, String format) {
+        try {
+            DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);
+            LocalDate.parse(value, formatter);
+            return true;
+        } catch (DateTimeParseException e) {
+            return false;
+        }
+    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo;
 
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import lombok.Data;
 
@@ -9,8 +10,8 @@ import java.util.List;
 public class MetricTable {
 
     private String alias;
-    private List<String> metrics;
-    private List<String> dimensions;
+    private List<String> metrics = Lists.newArrayList();
+    private List<String> dimensions = Lists.newArrayList();
     private String where;
     private AggOption aggOption = AggOption.DEFAULT;
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java

@@ -6,12 +6,11 @@ import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
+import lombok.Data;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import lombok.Data;
 
 @Data
 public class QueryParam {
@@ -34,7 +33,6 @@ public class QueryParam {
     // metric
     private List<String> metrics = new ArrayList();
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private List<ColumnOrder> order;
     private boolean nativeQuery = false;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java

@@ -5,7 +5,7 @@ package com.tencent.supersonic.headless.api.pojo.enums;
  * sql_query : view sql begin as select
  * table_query: dbName.tableName
  */
-public enum DatasourceQuery {
+public enum ModelDefineType {
 
     SQL_QUERY("sql_query"),
     TABLE_QUERY("table_query");
@@ -13,7 +13,7 @@ public enum DatasourceQuery {
     private String name;
 
 
-    DatasourceQuery(String name) {
+    ModelDefineType(String name) {
         this.name = name;
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java

@@ -5,14 +5,11 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import lombok.Builder;
 import lombok.Data;
 
-import javax.validation.constraints.NotNull;
-
 @Data
 @Builder
 public class DictItemFilter {
     private Long id;
     private TypeEnums type;
     private Long itemId;
-    @NotNull
     private StatusEnum status;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
+
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.Constants;
@@ -12,10 +13,6 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
@@ -39,6 +36,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 
 @Data
 @Slf4j
@@ -206,8 +208,7 @@ public class QueryStructReq extends SemanticQueryReq {
                     }
                     sumFunction.setParameters(new ExpressionList(new Column(columnName)));
                     SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
-                    String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
-                    selectExpressionItem.setAlias(new Alias(alias));
+                    selectExpressionItem.setAlias(new Alias(columnName));
                     selectItems.add(selectExpressionItem);
                 }
             }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryViewReq.java

@@ -0,0 +1,33 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.enums.QueryType;
+import com.tencent.supersonic.headless.api.pojo.Cache;
+import com.tencent.supersonic.headless.api.pojo.Param;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class QueryViewReq {
+
+    private Long viewId;
+    private String viewName;
+    private String sql;
+    private boolean needAuth = true;
+    private List<Param> params = new ArrayList<>();
+    private Cache cacheInfo = new Cache();
+    private List<String> groups = new ArrayList<>();
+    private List<Aggregator> aggregators = new ArrayList<>();
+    private List<Order> orders = new ArrayList<>();
+    private List<Filter> dimensionFilters = new ArrayList<>();
+    private List<Filter> metricFilters = new ArrayList<>();
+    private DateConf dateInfo;
+    private Long limit = 2000L;
+    private QueryType queryType = QueryType.ID;
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java

@@ -1,10 +1,12 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
 
-import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
+import com.tencent.supersonic.headless.api.pojo.SqlVariable;
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.util.List;
 
 @Data
 public class SqlExecuteReq {
@@ -16,6 +18,8 @@ public class SqlExecuteReq {
     @NotBlank(message = "sql can not be blank")
     private String sql;
 
+    private List<SqlVariable> sqlVariables;
+
     public String getSql() {
         if (StringUtils.isNotBlank(sql) && sql.endsWith(";")) {
             sql = sql.substring(0, sql.length() - 1);

headless/core/src/main/java/com/tencent/supersonic/headless/core/file/FileHandlerImpl.java

@@ -3,6 +3,8 @@ package com.tencent.supersonic.headless.core.file;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
+
+import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -17,9 +19,9 @@ import java.util.List;
 @Slf4j
 @Component
 public class FileHandlerImpl implements FileHandler {
+    public static final String FILE_SPILT = File.separator;
 
     private final LocalFileConfig localFileConfig;
-
     public FileHandlerImpl(LocalFileConfig localFileConfig) {
         this.localFileConfig = localFileConfig;
     }
@@ -31,8 +33,8 @@ public class FileHandlerImpl implements FileHandler {
             createDir(dictDirectoryBackup);
         }
 
-        String source = localFileConfig.getDictDirectoryLatest() + "/" + fileName;
-        String target = dictDirectoryBackup + "/" + fileName;
+        String source = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
+        String target = dictDirectoryBackup + FILE_SPILT + fileName;
         Path sourcePath = Paths.get(source);
         Path targetPath = Paths.get(target);
         try {
@@ -88,7 +90,7 @@ public class FileHandlerImpl implements FileHandler {
         if (!existPath(dictDirectoryLatest)) {
             createDir(dictDirectoryLatest);
         }
-        String filePath = dictDirectoryLatest + "/" + fileName;
+        String filePath = dictDirectoryLatest + FILE_SPILT + fileName;
         if (existPath(filePath)) {
             backupFile(fileName);
         }
@@ -117,7 +119,7 @@ public class FileHandlerImpl implements FileHandler {
     @Override
     public Boolean deleteDictFile(String fileName) {
         backupFile(fileName);
-        deleteFile(localFileConfig.getDictDirectoryLatest() + "/" + fileName);
+        deleteFile(localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName);
         return true;
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java

@@ -1,10 +1,13 @@
 package com.tencent.supersonic.headless.core.file;
 
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 
+import java.io.FileNotFoundException;
+
 @Data
 @Configuration
 @Slf4j
@@ -18,16 +21,21 @@ public class LocalFileConfig {
     private String dictDirectoryBackup;
 
     public String getDictDirectoryLatest() {
-        return getResourceDir() + dictDirectoryLatest;
+        return getDictDirectoryPrefixDir() + dictDirectoryLatest;
     }
 
     public String getDictDirectoryBackup() {
-        return getResourceDir() + dictDirectoryBackup;
+        return getDictDirectoryPrefixDir() + dictDirectoryBackup;
     }
 
-    private String getResourceDir() {
-        //return hanlpPropertiesPath = HanlpHelper.getHanlpPropertiesPath();
-        return ClassLoader.getSystemClassLoader().getResource("").getPath();
+    private String getDictDirectoryPrefixDir() {
+        try {
+            return HanlpHelper.getHanlpPropertiesPath();
+        } catch (FileNotFoundException e) {
+            log.warn("getDictDirectoryPrefixDir error: " + e);
+            e.printStackTrace();
+        }
+        return "";
     }
 
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java

@@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 /**
  * supplement the QueryStatement when query with custom aggregation method
@@ -110,7 +111,6 @@ public class CalculateAggConverter implements HeadlessConverter {
                 EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
         sqlCommend.setSql(viewQueryParam.getSql());
         sqlCommend.setTables(viewQueryParam.getTables());
-        sqlCommend.setVariables(viewQueryParam.getVariables());
         sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java

@@ -1,16 +1,25 @@
 package com.tencent.supersonic.headless.core.parser.converter;
 
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.headless.api.pojo.QueryParam;
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
+import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import lombok.extern.slf4j.Slf4j;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.StringValue;
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.expression.operators.relational.InExpression;
+import net.sf.jsqlparser.schema.Column;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 @Slf4j
 @Component("DefaultDimValueConverter")
@@ -18,36 +27,43 @@ public class DefaultDimValueConverter implements HeadlessConverter {
 
     @Override
     public boolean accept(QueryStatement queryStatement) {
-        if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
-            return false;
-        }
-        return true;
+        return !Objects.isNull(queryStatement.getViewQueryParam())
+                && !StringUtils.isBlank(queryStatement.getViewQueryParam().getSql());
     }
 
     @Override
     public void convert(QueryStatement queryStatement) {
-        QueryParam queryParam = queryStatement.getQueryParam();
         List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream()
                 .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
                 .collect(Collectors.toList());
         if (CollectionUtils.isEmpty(dimensions)) {
             return;
         }
-        log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryParam);
-        //add dimension default value to filter
-        List<String> dimensionFilterBizName = queryParam.getDimensionFilters().stream()
-                .map(Filter::getBizName).collect(Collectors.toList());
-        if (!CollectionUtils.isEmpty(dimensionFilterBizName)) {
+        String sql = queryStatement.getViewQueryParam().getSql();
+        List<String> whereFields = SqlSelectHelper.getWhereFields(sql)
+                .stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
+                .collect(Collectors.toList());
+        if (!CollectionUtils.isEmpty(whereFields)) {
             return;
         }
-        for (Dimension dimensionResp : dimensions) {
-            Filter filter = new Filter();
-            filter.setBizName(dimensionResp.getBizName());
-            filter.setValue(dimensionResp.getDefaultValues());
-            filter.setOperator(FilterOperatorEnum.IN);
-            filter.setName(dimensionResp.getName());
-            queryParam.getDimensionFilters().add(filter);
+        MetricTable metricTable = queryStatement.getViewQueryParam()
+                .getTables().stream().findFirst().orElse(null);
+        List<Expression> expressions = Lists.newArrayList();
+        for (Dimension dimension : dimensions) {
+            ExpressionList expressionList = new ExpressionList();
+            List<Expression> exprs = new ArrayList<>();
+            dimension.getDefaultValues().forEach(value -> exprs.add(new StringValue(value)));
+            expressionList.setExpressions(exprs);
+            InExpression inExpression = new InExpression();
+            inExpression.setLeftExpression(new Column(dimension.getBizName()));
+            inExpression.setRightItemsList(expressionList);
+            expressions.add(inExpression);
+            if (metricTable != null) {
+                metricTable.getDimensions().add(dimension.getBizName());
+            }
         }
+        sql = SqlAddHelper.addWhere(sql, expressions);
+        queryStatement.getViewQueryParam().setSql(sql);
     }
 
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java

@@ -1,20 +1,20 @@
 package com.tencent.supersonic.headless.core.parser.converter;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import com.tencent.supersonic.headless.api.pojo.Param;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
-import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 /**
  * HeadlessConverter default implement
  */
@@ -59,8 +59,6 @@ public class ParserDefaultConverter implements HeadlessConverter {
         metricQueryParam.setWhere(where);
         metricQueryParam.setOrder(queryParam.getOrders().stream()
                 .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
-        metricQueryParam.setVariables(queryParam.getParams().stream()
-                .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
         metricQueryParam.setLimit(queryParam.getLimit());
 
         // support detail query

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java

@@ -0,0 +1,49 @@
+package com.tencent.supersonic.headless.core.parser.converter;
+
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
+import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+@Slf4j
+@Component("SqlVariableParseConverter")
+public class SqlVariableParseConverter implements HeadlessConverter {
+
+    @Override
+    public boolean accept(QueryStatement queryStatement) {
+        if (Objects.isNull(queryStatement.getQueryParam())) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void convert(QueryStatement queryStatement) {
+        SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
+        List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
+        if (CollectionUtils.isEmpty(modelResps)) {
+            return;
+        }
+        for (ModelResp modelResp : modelResps) {
+            if (ModelDefineType.SQL_QUERY.getName()
+                    .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) {
+                String sqlParsed = SqlVariableParseUtils.parse(
+                        modelResp.getModelDetail().getSqlQuery(),
+                        modelResp.getModelDetail().getSqlVariables(),
+                        queryStatement.getQueryParam().getParams()
+                );
+                DataSource dataSource = queryStatement.getSemanticModel()
+                        .getDatasourceMap().get(modelResp.getBizName());
+                dataSource.setSqlQuery(sqlParsed);
+            }
+        }
+    }
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java

@@ -1,16 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
+import java.util.List;
 
 @Data
 public class MetricQueryParam {
 
     private List<String> metrics;
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private Long limit;
     private List<ColumnOrder> order;

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java

@@ -1,24 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ViewQueryParam {
-
-    private Map<String, String> variables;
     private String sql = "";
     private List<MetricTable> tables;
     private boolean supportWith = true;
     private boolean withAlias = true;
-
-    public Map<String, String> getVariables() {
-        if (variables == null) {
-            variables = new HashMap<>();
-        }
-        return variables;
-    }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.core.utils;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
+import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter;
 import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
 import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
@@ -83,6 +84,7 @@ public class ComponentFactory {
 
     private static void initSemanticConverter() {
         headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
+        headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class));
         headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
         headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java

@@ -30,13 +30,13 @@ public class SqlVariableParseUtils {
     private static final char delimiter = '$';
 
     public static String parse(String sql, List<SqlVariable> sqlVariables, List<Param> params) {
+        Map<String, Object> variables = new HashMap<>();
         if (CollectionUtils.isEmpty(sqlVariables)) {
             return sql;
         }
-        Map<String, Object> queryParams = new HashMap<>();
         //1. handle default variable value
         sqlVariables.forEach(variable -> {
-            queryParams.put(variable.getName().trim(),
+            variables.put(variable.getName().trim(),
                     getValues(variable.getValueType(), variable.getDefaultValues()));
         });
 
@@ -49,21 +49,25 @@ public class SqlVariableParseUtils {
                     List<SqlVariable> list = map.get(p.getName());
                     if (!CollectionUtils.isEmpty(list)) {
                         SqlVariable v = list.get(list.size() - 1);
-                        queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
+                        variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
                     }
                 }
             });
         }
 
-        queryParams.forEach((k, v) -> {
+        variables.forEach((k, v) -> {
             if (v instanceof List && ((List) v).size() > 0) {
                 v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
             }
-            queryParams.put(k, v);
+            variables.put(k, v);
         });
+        return parse(sql, variables);
+    }
+
+    public static String parse(String sql, Map<String, Object> variables) {
         ST st = new ST(sql, delimiter, delimiter);
-        if (!CollectionUtils.isEmpty(queryParams)) {
-            queryParams.forEach(st::add);
+        if (!CollectionUtils.isEmpty(variables)) {
+            variables.forEach(st::add);
         }
         return st.render();
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java

@@ -16,7 +16,7 @@ import java.util.List;
 public class SysTimeDimensionBuilder {
 
     public static void addSysTimeDimension(List<Dim> dims, DbAdaptor engineAdaptor) {
-        log.info("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
+        log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
         Dim timeDim = getTimeDim(dims);
         if (timeDim == null) {
             timeDim = Dim.getDefault();

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -4,7 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
-import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery;
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
@@ -46,7 +46,7 @@ public class ModelYamlManager {
                 .collect(Collectors.toList()));
         dataModelYamlTpl.setName(modelResp.getBizName());
         dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
-        if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
+        if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {
             dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
         } else {
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
 
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
@@ -17,16 +16,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.DictReposit
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Repository
 public class DictRepositoryImpl implements DictRepository {
 
+    @Value("${dict.task.num:10}")
+    private Integer dictTaskNum;
+
     private final DictTaskMapper dictTaskMapper;
     private final DictConfMapper dictConfMapper;
     private final DictUtils dictConverter;
@@ -83,7 +88,9 @@ public class DictRepositoryImpl implements DictRepository {
         QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
         wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
         wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
-        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper);
+        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
+                .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
+                .limit(dictTaskNum).collect(Collectors.toList());
         if (CollectionUtils.isEmpty(dictTaskDOList)) {
             return taskResp;
         }
@@ -102,9 +109,9 @@ public class DictRepositoryImpl implements DictRepository {
 
     @Override
     public Long editDictConf(DictConfDO dictConfDO) {
-        DictItemFilter filter = DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType()))
+        DictItemFilter filter = DictItemFilter.builder()
+                .type(TypeEnums.valueOf(dictConfDO.getType()))
                 .itemId(dictConfDO.getItemId())
-                .status(StatusEnum.ONLINE)
                 .build();
 
         List<DictConfDO> dictConfDOList = getDictConfDOList(filter);

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java

@@ -73,7 +73,7 @@ public class DatabaseController {
             HttpServletRequest request,
             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return databaseService.executeSql(sqlExecuteReq.getSql(), sqlExecuteReq.getId(), user);
+        return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
     }
 
     @RequestMapping("/getDbNames/{id}")

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java

@@ -42,7 +42,7 @@ public class KnowledgeController {
      * @param dictItemReq
      */
     @PostMapping("/conf")
-    public Long addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                             HttpServletRequest request,
                             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
@@ -56,7 +56,7 @@ public class KnowledgeController {
      * @param dictItemReq
      */
     @PutMapping("/conf")
-    public Long editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                              HttpServletRequest request,
                              HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java

@@ -13,7 +13,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
 import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
 import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
 import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
@@ -42,14 +41,6 @@ public class QueryController {
     @Autowired
     private DownloadService downloadService;
 
-    @PostMapping("/sql")
-    public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
-            HttpServletRequest request,
-            HttpServletResponse response) throws Exception {
-        User user = UserHolder.findUser(request, response);
-        return queryService.queryByReq(querySqlReq, user);
-    }
-
     @PostMapping("/struct")
     public Object queryByStruct(@RequestBody QueryStructReq queryStructReq,
             HttpServletRequest request,
@@ -59,14 +50,6 @@ public class QueryController {
         return queryService.queryByReq(querySqlReq, user);
     }
 
-    @PostMapping("/tag")
-    public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
-            HttpServletRequest request,
-            HttpServletResponse response) throws Exception {
-        User user = UserHolder.findUser(request, response);
-        return queryService.queryByReq(queryTagReq, user);
-    }
-
     @PostMapping("/queryMetricDataById")
     public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
             HttpServletRequest request) throws Exception {

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetricQueryApiController.java

@@ -0,0 +1,38 @@
+package com.tencent.supersonic.headless.server.rest.api;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.server.service.MetricService;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class MetricQueryApiController {
+
+    @Autowired
+    private QueryService queryService;
+
+    @Autowired
+    private MetricService metricService;
+
+    @PostMapping("/metric")
+    public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
+            HttpServletRequest request,
+            HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
+        return queryService.queryByReq(queryStructReq.convert(), user);
+    }
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java

@@ -0,0 +1,31 @@
+package com.tencent.supersonic.headless.server.rest.api;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class SqlQueryApiController {
+
+    @Autowired
+    private QueryService queryService;
+
+    @PostMapping("/sql")
+    public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
+            HttpServletRequest request,
+            HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        return queryService.queryByReq(querySqlReq, user);
+    }
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java

@@ -1,8 +1,8 @@
-package com.tencent.supersonic.headless.server.rest;
+package com.tencent.supersonic.headless.server.rest.api;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
 import com.tencent.supersonic.headless.server.service.QueryService;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -16,17 +16,17 @@ import org.springframework.web.bind.annotation.RestController;
 @RestController
 @RequestMapping("/api/semantic/query")
 @Slf4j
-public class MetricApiController {
+public class TagQueryApiController {
 
     @Autowired
     private QueryService queryService;
 
-    @PostMapping("/metric")
-    public Object queryBySql(@RequestBody QueryMetricReq queryMetricReq,
+    @PostMapping("/tag")
+    public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
             HttpServletRequest request,
             HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        return queryService.queryByMetric(queryMetricReq, user);
+        return queryService.queryByReq(queryTagReq, user);
     }
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java

@@ -0,0 +1,37 @@
+package com.tencent.supersonic.headless.server.rest.api;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
+import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import com.tencent.supersonic.headless.server.service.ViewService;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class ViewQueryApiController {
+
+    @Autowired
+    private ViewService viewService;
+    @Autowired
+    private QueryService queryService;
+
+    @PostMapping("/view")
+    public Object queryByView(@RequestBody QueryViewReq queryViewReq,
+            HttpServletRequest request,
+            HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        SemanticQueryReq queryReq = viewService.convert(queryViewReq);
+        return queryService.queryByReq(queryReq, user);
+    }
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
+import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
@@ -13,7 +14,7 @@ public interface DatabaseService {
 
     SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp);
 
-    SemanticQueryResp executeSql(String sql, Long id, User user);
+    SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user);
 
     DatabaseResp getDatabase(Long id, User user);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java

@@ -12,9 +12,9 @@ import java.util.List;
  */
 public interface DictConfService {
 
-    Long addDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp addDictConf(DictItemReq itemValueReq, User user);
 
-    Long editDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp editDictConf(DictItemReq itemValueReq, User user);
 
     List<DictItemResp> queryDictConf(DictItemFilter dictItemFilter, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java

@@ -9,6 +9,8 @@ import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
@@ -52,4 +54,6 @@ public interface MetricService {
     void sendMetricEventBatch(List<Long> modelIds, EventType eventType);
 
     List<MetricResp> queryMetrics(MetricsFilter metricsFilter);
+
+    QueryStructReq convert(QueryMetricReq queryMetricReq);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java

@@ -5,7 +5,6 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
 import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
@@ -28,6 +27,4 @@ public interface QueryService {
     @ApiHeaderCheck
     ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
 
-    SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception;
-
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java

@@ -6,7 +6,6 @@ import com.tencent.supersonic.headless.api.pojo.request.TagReq;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
-
 import java.util.List;
 
 public interface TagService {
@@ -22,4 +21,5 @@ public interface TagService {
     List<TagResp> query(TagFilter tagFilter);
 
     PageInfo<TagResp> queryPage(TagFilterPage tagFilterPage, User user);
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java

@@ -1,6 +1,8 @@
 package com.tencent.supersonic.headless.server.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
+import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
 import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
@@ -25,4 +27,6 @@ public interface ViewService {
     List<ViewResp> getViews(User user);
 
     List<ViewResp> getViewsInheritAuth(User user, Long domainId);
+
+    SemanticQueryReq convert(QueryViewReq queryViewReq);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -1,8 +1,9 @@
 package com.tencent.supersonic.headless.server.service.impl;
 
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
+import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
@@ -11,6 +12,7 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
 import com.tencent.supersonic.headless.core.utils.SqlUtils;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
 import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
@@ -116,32 +118,19 @@ public class DatabaseServiceImpl implements DatabaseService {
     @Override
     public DatabaseResp getDatabase(Long id, User user) {
         DatabaseResp databaseResp = getDatabase(id);
-        if (!databaseResp.getAdmins().contains(user.getName())
-                && !databaseResp.getViewers().contains(user.getName())
-                && !databaseResp.getCreatedBy().equals(user.getName())) {
-            throw new InvalidPermissionException("您暂无查看该数据库详情的权限, 请联系创建人: "
-                    + databaseResp.getCreatedBy());
-        }
+        checkPermission(databaseResp, user);
         return databaseResp;
     }
 
     @Override
-    public SemanticQueryResp executeSql(String sql, Long id, User user) {
+    public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user) {
         DatabaseResp databaseResp = getDatabase(id);
         if (databaseResp == null) {
             return new SemanticQueryResp();
         }
-        List<String> admins = databaseResp.getAdmins();
-        List<String> viewers = databaseResp.getViewers();
-        if (!admins.contains(user.getName())
-                && !viewers.contains(user.getName())
-                && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
-                && !user.isSuperAdmin()) {
-            String message = String.format("您暂无当前数据库%s权限, 请联系数据库管理员%s开通",
-                    databaseResp.getName(),
-                    String.join(",", admins));
-            throw new RuntimeException(message);
-        }
+        checkPermission(databaseResp, user);
+        String sql = sqlExecuteReq.getSql();
+        sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList());
         return executeSql(sql, databaseResp);
     }
 
@@ -195,4 +184,18 @@ public class DatabaseServiceImpl implements DatabaseService {
         return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
     }
 
+    private void checkPermission(DatabaseResp databaseResp, User user) {
+        List<String> admins = databaseResp.getAdmins();
+        List<String> viewers = databaseResp.getViewers();
+        if (!admins.contains(user.getName())
+                && !viewers.contains(user.getName())
+                && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
+                && !user.isSuperAdmin()) {
+            String message = String.format("您暂无当前数据库%s权限, 请联系数据库创建人:%s开通",
+                    databaseResp.getName(),
+                    databaseResp.getCreatedBy());
+            throw new RuntimeException(message);
+        }
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
@@ -10,11 +9,15 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
 import com.tencent.supersonic.headless.server.service.DictConfService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Optional;
 
 @Service
+@Slf4j
 public class DictConfServiceImpl implements DictConfService {
 
     private final DictRepository dictRepository;
@@ -27,20 +30,49 @@ public class DictConfServiceImpl implements DictConfService {
     }
 
     @Override
-    public Long addDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp addDictConf(DictItemReq itemValueReq, User user) {
         DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
-        return dictRepository.addDictConf(dictConfDO);
+        Boolean exist = checkConfExist(itemValueReq, user);
+        if (exist) {
+            throw new RuntimeException("dictConf is existed");
+        }
+        Long id = dictRepository.addDictConf(dictConfDO);
+        log.debug("dictConfDO:{}", dictConfDO);
+
+        DictItemFilter filter = DictItemFilter.builder()
+                .id(id)
+                .status(itemValueReq.getStatus())
+                .build();
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
+        }
+        return null;
+    }
+
+    private Boolean checkConfExist(DictItemReq itemValueReq, User user) {
+        DictItemFilter filter = DictItemFilter.builder().build();
+        BeanUtils.copyProperties(itemValueReq, filter);
+        filter.setStatus(null);
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream()
+                .findFirst();
+        if (dictItemResp.isPresent()) {
+            return true;
+        }
+        return false;
     }
 
     @Override
-    public Long editDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp editDictConf(DictItemReq itemValueReq, User user) {
         DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
         dictRepository.editDictConf(dictConfDO);
-        if (StatusEnum.DELETED.equals(itemValueReq.getStatus())) {
-            // todo delete dict file and refresh
-
+        DictItemFilter filter = DictItemFilter.builder().build();
+        BeanUtils.copyProperties(itemValueReq, filter);
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
         }
-        return itemValueReq.getItemId();
+        return null;
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java

@@ -65,7 +65,7 @@ public class DictTaskServiceImpl implements DictTaskService {
     }
 
     private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) {
-        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user);
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING);
         log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
         dictRepository.addDictTask(dictTaskDO);
         Long idInDb = dictTaskDO.getId();
@@ -95,14 +95,14 @@ public class DictTaskServiceImpl implements DictTaskService {
         dictTaskDO.setStatus(TaskStatusEnum.RUNNING.getStatus());
         dictRepository.editDictTask(dictTaskDO);
 
-        // 1.生成item字典数据
+        // 1.Generate item dictionary data
         List<String> data = dictUtils.fetchItemValue(dictItemResp);
 
-        // 2.变更字典文件
+        // 2.Change dictionary file
         String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType;
         fileHandler.writeFile(data, fileName, false);
 
-        // 3.实时变更内存中字典数据
+        // 3.Change in-memory dictionary data in real time
         try {
             HanlpHelper.reloadCustomDictionary();
             dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
@@ -124,7 +124,10 @@ public class DictTaskServiceImpl implements DictTaskService {
         } catch (Exception e) {
             log.error("reloadCustomDictionary error", e);
         }
-
+        // Add a clear dictionary file record
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL);
+        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
+        dictRepository.addDictTask(dictTaskDO);
         return 0L;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -7,6 +7,7 @@ import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DataEvent;
 import com.tencent.supersonic.common.pojo.DataItem;
@@ -20,11 +21,15 @@ import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricParam;
 import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
+import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
@@ -32,23 +37,22 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
 import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
+import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
+import com.tencent.supersonic.headless.server.pojo.ModelCluster;
 import com.tencent.supersonic.headless.server.service.CollectService;
-import com.tencent.supersonic.headless.server.service.DomainService;
+import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.ViewService;
 import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.BeanUtils;
-import org.springframework.context.ApplicationEventPublisher;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
+import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -56,6 +60,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 
 @Service
 @Slf4j
@@ -65,7 +74,7 @@ public class MetricServiceImpl implements MetricService {
 
     private ModelService modelService;
 
-    private DomainService domainService;
+    private DimensionService dimensionService;
 
     private ChatGptHelper chatGptHelper;
 
@@ -77,18 +86,18 @@ public class MetricServiceImpl implements MetricService {
 
     public MetricServiceImpl(MetricRepository metricRepository,
             ModelService modelService,
-            DomainService domainService,
             ChatGptHelper chatGptHelper,
             CollectService collectService,
             ViewService viewService,
-            ApplicationEventPublisher eventPublisher) {
-        this.domainService = domainService;
+            ApplicationEventPublisher eventPublisher,
+            DimensionService dimensionService) {
         this.metricRepository = metricRepository;
         this.modelService = modelService;
         this.chatGptHelper = chatGptHelper;
         this.eventPublisher = eventPublisher;
         this.collectService = collectService;
         this.viewService = viewService;
+        this.dimensionService = dimensionService;
     }
 
     @Override
@@ -239,7 +248,7 @@ public class MetricServiceImpl implements MetricService {
     }
 
     private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
-                                  List<String> fields, Set<MetricResp> metricRespFiltered) {
+            List<String> fields, Set<MetricResp> metricRespFiltered) {
         if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
             List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
                     .stream().map(MetricParam::getId).collect(Collectors.toList());
@@ -483,4 +492,107 @@ public class MetricServiceImpl implements MetricService {
                 .type(TypeEnums.METRIC).defaultAgg(metricResp.getDefaultAgg()).build();
     }
 
+    public QueryStructReq convert(QueryMetricReq queryMetricReq) {
+        //1. If a domainId exists, the modelIds obtained from the domainId.
+        Set<Long> modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq);
+
+        //2. get metrics and dimensions
+        List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
+
+        List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
+
+        //3. choose ModelCluster
+        Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
+        ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
+
+        //4. set groups
+        List<String> dimensionBizNames = dimensionResps.stream()
+                .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
+                .map(entry -> entry.getBizName()).collect(Collectors.toList());
+
+        QueryStructReq queryStructReq = new QueryStructReq();
+        if (org.apache.commons.collections.CollectionUtils.isNotEmpty(dimensionBizNames)) {
+            queryStructReq.setGroups(dimensionBizNames);
+        }
+        //5. set aggregators
+        List<String> metricBizNames = metricResps.stream()
+                .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
+                .map(entry -> entry.getBizName()).collect(Collectors.toList());
+        if (org.apache.commons.collections.CollectionUtils.isEmpty(metricBizNames)) {
+            throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
+        }
+        List<Aggregator> aggregators = new ArrayList<>();
+        for (String metricBizName : metricBizNames) {
+            Aggregator aggregator = new Aggregator();
+            aggregator.setColumn(metricBizName);
+            aggregators.add(aggregator);
+        }
+        queryStructReq.setAggregators(aggregators);
+        queryStructReq.setLimit(queryMetricReq.getLimit());
+        //6. set modelIds
+        for (Long modelId : modelCluster.getModelIds()) {
+            queryStructReq.addModelId(modelId);
+        }
+        //7. set dateInfo
+        queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
+        return queryStructReq;
+    }
+
+    private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> modelIds) {
+        Map<String, ModelCluster> modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds));
+
+        Map<String, List<SchemaItem>> modelClusterToMatchCount = new HashMap<>();
+        for (ModelCluster modelCluster : modelClusterMap.values()) {
+            for (MetricResp metricResp : metricResps) {
+                if (modelCluster.getModelIds().contains(metricResp.getModelId())) {
+                    modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>())
+                            .add(metricResp);
+                }
+            }
+        }
+        String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
+                .max(Comparator.comparingInt(entry -> entry.getValue().size()))
+                .map(Map.Entry::getKey)
+                .orElse(null);
+
+        return modelClusterMap.get(keyWithMaxSize);
+    }
+
+    private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
+            List<DimensionResp> dimensionResps) {
+        Set<Long> result = new HashSet<>();
+        if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
+            result.addAll(modelIdsByDomainId);
+            return result;
+        }
+        Set<Long> metricModelIds = metricResps.stream().map(entry -> entry.getModelId())
+                .collect(Collectors.toSet());
+        result.addAll(metricModelIds);
+
+        Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
+                .collect(Collectors.toSet());
+        result.addAll(dimensionModelIds);
+        return result;
+    }
+
+    private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
+        DimensionsFilter dimensionsFilter = new DimensionsFilter();
+        BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
+        dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
+        return dimensionService.queryDimensions(dimensionsFilter);
+    }
+
+    private List<MetricResp> getMetricResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
+        MetricsFilter metricsFilter = new MetricsFilter();
+        BeanUtils.copyProperties(queryMetricReq, metricsFilter);
+        metricsFilter.setModelIds(new ArrayList<>(modelIds));
+        return queryMetrics(metricsFilter);
+    }
+
+    private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
+        List<ModelResp> modelResps = modelService.getAllModelByDomainIds(
+                Collections.singletonList(queryMetricReq.getDomainId()));
+        return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java

@@ -13,13 +13,11 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.Item;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
 import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
-import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -46,25 +44,14 @@ import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
 import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
 import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
-import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
-import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
-import com.tencent.supersonic.headless.server.pojo.ModelCluster;
 import com.tencent.supersonic.headless.server.service.AppService;
 import com.tencent.supersonic.headless.server.service.Catalog;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
-import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
 import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
 import com.tencent.supersonic.headless.server.utils.StatUtils;
 import com.tencent.supersonic.headless.server.utils.TagReqConverter;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -74,7 +61,6 @@ import javax.servlet.http.HttpServletRequest;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 
@@ -90,17 +76,9 @@ public class QueryServiceImpl implements QueryService {
     private final AppService appService;
     private final QueryCache queryCache;
     private final SemanticSchemaManager semanticSchemaManager;
-
     private final QueryParser queryParser;
-
     private final QueryPlanner queryPlanner;
 
-    private final MetricService metricService;
-
-    private final ModelService modelService;
-
-    private final DimensionService dimensionService;
-
     public QueryServiceImpl(
             StatUtils statUtils,
             QueryUtils queryUtils,
@@ -110,10 +88,7 @@ public class QueryServiceImpl implements QueryService {
             QueryCache queryCache,
             SemanticSchemaManager semanticSchemaManager,
             DefaultQueryParser queryParser,
-            QueryPlanner queryPlanner,
-            MetricService metricService,
-            ModelService modelService,
-            DimensionService dimensionService) {
+            QueryPlanner queryPlanner) {
         this.statUtils = statUtils;
         this.queryUtils = queryUtils;
         this.queryReqConverter = queryReqConverter;
@@ -124,9 +99,6 @@ public class QueryServiceImpl implements QueryService {
         this.semanticSchemaManager = semanticSchemaManager;
         this.queryParser = queryParser;
         this.queryPlanner = queryPlanner;
-        this.metricService = metricService;
-        this.modelService = modelService;
-        this.dimensionService = dimensionService;
     }
 
     @Override
@@ -287,58 +259,6 @@ public class QueryServiceImpl implements QueryService {
         return ItemQueryResultResp.builder().results(results).build();
     }
 
-    @Override
-    public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) {
-        QueryStructReq queryStructReq = buildQueryStructReq(queryMetricReq);
-        return queryByReq(queryStructReq.convert(), user);
-    }
-
-    private QueryStructReq buildQueryStructReq(QueryMetricReq queryMetricReq) {
-        //1. If a domainId exists, the modelIds obtained from the domainId.
-        Set<Long> modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq);
-
-        //2. get metrics and dimensions
-        List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
-
-        List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
-
-        //3. choose ModelCluster
-        Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
-        ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
-
-        //4. set groups
-        List<String> dimensionBizNames = dimensionResps.stream()
-                .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
-                .map(entry -> entry.getBizName()).collect(Collectors.toList());
-
-        QueryStructReq queryStructReq = new QueryStructReq();
-        if (CollectionUtils.isNotEmpty(dimensionBizNames)) {
-            queryStructReq.setGroups(dimensionBizNames);
-        }
-        //5. set aggregators
-        List<String> metricBizNames = metricResps.stream()
-                .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
-                .map(entry -> entry.getBizName()).collect(Collectors.toList());
-        if (CollectionUtils.isEmpty(metricBizNames)) {
-            throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
-        }
-        List<Aggregator> aggregators = new ArrayList<>();
-        for (String metricBizName : metricBizNames) {
-            Aggregator aggregator = new Aggregator();
-            aggregator.setColumn(metricBizName);
-            aggregators.add(aggregator);
-        }
-        queryStructReq.setAggregators(aggregators);
-        queryStructReq.setLimit(queryMetricReq.getLimit());
-        //6. set modelIds
-        for (Long modelId : modelCluster.getModelIds()) {
-            queryStructReq.addModelId(modelId);
-        }
-        //7. set dateInfo
-        queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
-        return queryStructReq;
-    }
-
     private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
             MetricResp metricResp, DateConf dateConf, Long limit) {
         Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
@@ -356,64 +276,6 @@ public class QueryServiceImpl implements QueryService {
         return queryStructReq;
     }
 
-    private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> modelIds) {
-        Map<String, ModelCluster> modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds));
-
-        Map<String, List<SchemaItem>> modelClusterToMatchCount = new HashMap<>();
-        for (ModelCluster modelCluster : modelClusterMap.values()) {
-            for (MetricResp metricResp : metricResps) {
-                if (modelCluster.getModelIds().contains(metricResp.getModelId())) {
-                    modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>())
-                            .add(metricResp);
-                }
-            }
-        }
-        String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
-                .max(Comparator.comparingInt(entry -> entry.getValue().size()))
-                .map(Map.Entry::getKey)
-                .orElse(null);
-
-        return modelClusterMap.get(keyWithMaxSize);
-    }
-
-    private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
-            List<DimensionResp> dimensionResps) {
-        Set<Long> result = new HashSet<>();
-        if (CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
-            result.addAll(modelIdsByDomainId);
-            return result;
-        }
-        Set<Long> metricModelIds = metricResps.stream().map(entry -> entry.getModelId())
-                .collect(Collectors.toSet());
-        result.addAll(metricModelIds);
-
-        Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
-                .collect(Collectors.toSet());
-        result.addAll(dimensionModelIds);
-        return result;
-    }
-
-    private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
-        DimensionsFilter dimensionsFilter = new DimensionsFilter();
-        BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
-        dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
-        List<DimensionResp> dimensionResps = dimensionService.queryDimensions(dimensionsFilter);
-        return dimensionResps;
-    }
-
-    private List<MetricResp> getMetricResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
-        MetricsFilter metricsFilter = new MetricsFilter();
-        BeanUtils.copyProperties(queryMetricReq, metricsFilter);
-        metricsFilter.setModelIds(new ArrayList<>(modelIds));
-        return metricService.queryMetrics(metricsFilter);
-    }
-
-    private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
-        List<ModelResp> modelResps = modelService.getAllModelByDomainIds(
-                Collections.singletonList(queryMetricReq.getDomainId()));
-        return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
-    }
-
     private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
         MetricResp metricResp = catalog.getMetric(item.getId());
         item.setCreatedBy(metricResp.getCreatedBy());

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
+
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
@@ -23,7 +24,6 @@ import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -32,7 +32,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -183,6 +182,7 @@ public class TagServiceImpl implements TagService {
     private void checkExit(TagReq tagReq) {
         TagFilter tagFilter = new TagFilter();
         tagFilter.setModelIds(Arrays.asList(tagReq.getModelId()));
+
         List<TagResp> tagResps = query(tagFilter);
         if (!CollectionUtils.isEmpty(tagResps)) {
             Long bizNameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId()))

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java

@@ -14,6 +14,10 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.headless.api.pojo.QueryConfig;
 import com.tencent.supersonic.headless.api.pojo.ViewDetail;
+import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
+import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
 import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
@@ -28,6 +32,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ViewService;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
@@ -174,6 +179,15 @@ public class ViewServiceImpl
         return viewDO;
     }
 
+    public SemanticQueryReq convert(QueryViewReq queryViewReq) {
+        SemanticQueryReq queryReq = new QueryStructReq();
+        if (StringUtils.isNotBlank(queryViewReq.getSql())) {
+            queryReq = new QuerySqlReq();
+        }
+        BeanUtils.copyProperties(queryViewReq, queryReq);
+        return queryReq;
+    }
+
     public static boolean checkAdminPermission(User user, ViewResp viewResp) {
         List<String> admins = viewResp.getAdmins();
         if (user.isSuperAdmin()) {
@@ -239,5 +253,4 @@ public class ViewServiceImpl
                 .map(Object::toString)
                 .collect(Collectors.toList());
     }
-
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils;
 import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
 import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
 import static com.tencent.supersonic.common.pojo.Constants.COMMA;
+import static com.tencent.supersonic.common.pojo.Constants.POUND;
 import static com.tencent.supersonic.common.pojo.Constants.SPACE;
 
 import com.google.common.base.Strings;
@@ -36,6 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -48,6 +50,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
+
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -78,9 +81,9 @@ public class DictUtils {
     private final ModelService modelService;
 
     public DictUtils(DimensionService dimensionService,
-            MetricService metricService,
-            QueryService queryService,
-            ModelService modelService) {
+                     MetricService metricService,
+                     QueryService queryService,
+                     ModelService modelService) {
         this.dimensionService = dimensionService;
         this.metricService = metricService;
         this.queryService = queryService;
@@ -92,7 +95,7 @@ public class DictUtils {
                 dictItemResp.getItemId());
     }
 
-    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user) {
+    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user, TaskStatusEnum status) {
         DictTaskDO taskDO = new DictTaskDO();
         Date createAt = new Date();
         String name = dictItemResp.fetchDictFileName();
@@ -100,7 +103,7 @@ public class DictUtils {
         taskDO.setType(dictItemResp.getType().name());
         taskDO.setItemId(dictItemResp.getItemId());
         taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig()));
-        taskDO.setStatus(TaskStatusEnum.PENDING.getStatus());
+        taskDO.setStatus(status.getStatus());
         taskDO.setCreatedAt(createAt);
         String creator = (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) ? "" : user.getName();
         taskDO.setCreatedBy(creator);
@@ -185,7 +188,12 @@ public class DictUtils {
             return;
         }
         List<String> whiteList = dictItemResp.getConfig().getWhiteList();
-        whiteList.forEach(white -> lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)));
+        whiteList.forEach(white -> {
+            if (!Strings.isNullOrEmpty(white)) {
+                white = white.replace(SPACE, POUND);
+            }
+            lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
+        });
     }
 
     private void constructDictLines(Map<String, Long> valueAndFrequencyPair, List<String> lines, String nature) {
@@ -194,6 +202,9 @@ public class DictUtils {
         }
 
         valueAndFrequencyPair.forEach((value, frequency) -> {
+            if (!Strings.isNullOrEmpty(value)) {
+                value = value.replace(SPACE, POUND);
+            }
             lines.add(String.format("%s %s %s", value, nature, frequency));
         });
     }

headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java

@@ -1,5 +1,8 @@
 package com.tencent.supersonic.headless.server.service;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.DataFormat;
@@ -8,12 +11,12 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.ChatGptHelper;
-import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
-import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
+import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
 import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
 import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
@@ -22,13 +25,11 @@ import com.tencent.supersonic.headless.server.persistence.repository.MetricRepos
 import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
 import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
+import java.util.HashMap;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 import org.springframework.context.ApplicationEventPublisher;
-import java.util.HashMap;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
 
 public class MetricServiceImplTest {
 
@@ -61,14 +62,14 @@ public class MetricServiceImplTest {
     }
 
     private MetricService mockMetricService(MetricRepository metricRepository,
-                                            ModelService modelService) {
-        DomainService domainService = Mockito.mock(DomainService.class);
+            ModelService modelService) {
         ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
         CollectService collectService = Mockito.mock(CollectService.class);
         ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
         ViewService viewService = Mockito.mock(ViewServiceImpl.class);
-        return new MetricServiceImpl(metricRepository, modelService, domainService,
-                chatGptHelper, collectService, viewService, eventPublisher);
+        DimensionService dimensionService = Mockito.mock(DimensionService.class);
+        return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, viewService,
+                eventPublisher, dimensionService);
     }
 
     private MetricReq buildMetricReq() {
@@ -96,7 +97,7 @@ public class MetricServiceImplTest {
                 RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
                         new DrillDownDimension(1L),
                         new DrillDownDimension(1L, false))
-        ).build());
+                ).build());
         metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
         metricReq.setExt(new HashMap<>());
         return metricReq;

launchers/chat/src/main/resources/META-INF/spring.factories

@@ -14,10 +14,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
 com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
     com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
     com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
-    com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
-    com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
-    com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
-    com.tencent.supersonic.chat.core.corrector.HavingCorrector
+    com.tencent.supersonic.chat.core.corrector.GrammarCorrector
 
 com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\
     com.tencent.supersonic.chat.core.query.semantic.RemoteSemanticInterpreter

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -14,10 +14,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
 com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
     com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
     com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
-    com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
-    com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
-    com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
-    com.tencent.supersonic.chat.core.corrector.HavingCorrector
+    com.tencent.supersonic.chat.core.corrector.GrammarCorrector
 
 com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
     com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \

launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/PluginRecognizeTest.java

@@ -57,7 +57,7 @@ public class PluginRecognizeTest extends BasePluginTest {
         QueryFilter queryFilter = new QueryFilter();
         queryFilter.setElementID(2L);
         queryFilter.setValue("alice");
-        queryRequest.setModelId(1L);
+        // queryRequest.setModelId(1L);
         queryFilters.getFilters().add(queryFilter);
         queryRequest.setQueryFilters(queryFilters);
 

launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java

@@ -4,19 +4,25 @@ import static org.junit.Assert.assertThrows;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.server.service.MetricService;
 import java.util.Arrays;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 
 public class QueryByMetricTest extends BaseTest {
 
+    @Autowired
+    protected MetricService metricService;
+
     @Test
     public void testWithMetricAndDimensionBizNames() throws Exception {
         QueryMetricReq queryMetricReq = new QueryMetricReq();
         queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
         queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
-        SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
+        SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
         Assert.assertNotNull(queryResp.getResultList());
         Assert.assertEquals(6, queryResp.getResultList().size());
     }
@@ -26,7 +32,7 @@ public class QueryByMetricTest extends BaseTest {
         QueryMetricReq queryMetricReq = new QueryMetricReq();
         queryMetricReq.setMetricNames(Arrays.asList("停留时长", "访问次数"));
         queryMetricReq.setDimensionNames(Arrays.asList("用户", "部门"));
-        SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
+        SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
         Assert.assertNotNull(queryResp.getResultList());
         Assert.assertEquals(6, queryResp.getResultList().size());
     }
@@ -37,7 +43,7 @@ public class QueryByMetricTest extends BaseTest {
         queryMetricReq.setDomainId(1L);
         queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
         queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
-        SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
+        SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
         Assert.assertNotNull(queryResp.getResultList());
         Assert.assertEquals(6, queryResp.getResultList().size());
 
@@ -45,7 +51,7 @@ public class QueryByMetricTest extends BaseTest {
         queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
         queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
         assertThrows(IllegalArgumentException.class,
-                () -> queryService.queryByMetric(queryMetricReq, User.getFakeUser()));
+                () -> queryByMetric(queryMetricReq, User.getFakeUser()));
     }
 
     @Test
@@ -54,9 +60,13 @@ public class QueryByMetricTest extends BaseTest {
         queryMetricReq.setDomainId(1L);
         queryMetricReq.setMetricIds(Arrays.asList(1L, 4L));
         queryMetricReq.setDimensionIds(Arrays.asList(1L, 2L));
-        SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
+        SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
         Assert.assertNotNull(queryResp.getResultList());
         Assert.assertEquals(6, queryResp.getResultList().size());
     }
 
+    private SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception {
+        QueryStructReq convert = metricService.convert(queryMetricReq);
+        return queryService.queryByReq(convert.convert(), user);
+    }
 }

launchers/standalone/src/test/resources/META-INF/spring.factories

@@ -13,10 +13,8 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
 
 com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
     com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
-    com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
-    com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
-    com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
-    com.tencent.supersonic.chat.core.corrector.HavingCorrector
+    com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
+    com.tencent.supersonic.chat.core.corrector.GrammarCorrector
 
 com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
     com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \

launchers/standalone/src/test/resources/db/schema-h2.sql

@@ -529,4 +529,24 @@ CREATE TABLE IF NOT EXISTS `s2_view` (
     query_config VARCHAR(3000),
     `admin` varchar(3000) DEFAULT NULL,
     `admin_org` varchar(3000) DEFAULT NULL
-);
+);
+
+CREATE TABLE IF NOT EXISTS `s2_tag` (
+    `id` INT NOT NULL  AUTO_INCREMENT,
+    `model_id` INT  NOT NULL ,
+    `name` varchar(255)  NOT NULL ,
+    `biz_name` varchar(255)  NOT NULL ,
+    `description` varchar(500) DEFAULT NULL ,
+    `status` INT  NOT NULL ,
+    `sensitive_level` INT NOT NULL ,
+    `type` varchar(50)  NOT NULL , -- ATOMIC, DERIVED
+    `define_type` varchar(50)  NOT NULL, -- FIELD, DIMENSION
+    `type_params` LONGVARCHAR DEFAULT NULL  ,
+    `created_at` TIMESTAMP NOT NULL ,
+    `created_by` varchar(100) NOT NULL ,
+    `updated_at` TIMESTAMP DEFAULT NULL ,
+    `updated_by` varchar(100) DEFAULT NULL ,
+    `ext` LONGVARCHAR DEFAULT NULL  ,
+    PRIMARY KEY (`id`)
+    );
+COMMENT ON TABLE s2_tag IS 'tag information';

webapp/packages/supersonic-fe/src/enum/index.ts

@@ -1,22 +1,4 @@
 export * from './models/base';
-type ObjToArrayParams = Record<string, string>;
-
-const keyTypeTran = {
-  string: String,
-  number: Number,
-};
-/**
- * obj转成value，label的数组
- * @param _obj
- */
-export const objToArray = (_obj: ObjToArrayParams, keyType: string = 'string') => {
-  return Object.keys(_obj).map((key) => {
-    return {
-      value: keyTypeTran[keyType](key),
-      label: _obj[key],
-    };
-  });
-};
 
 type EnumToArrayItem = {
   value: number;

webapp/packages/supersonic-fe/src/pages/SemanticModel/ChatSetting/ChatSetting.tsx

@@ -1,21 +0,0 @@
-import React from 'react';
-import { connect } from 'umi';
-import type { StateType } from '../model';
-import OverviewContainer from '../OverviewContainer';
-import type { Dispatch } from 'umi';
-
-type Props = {
-  domainManger: StateType;
-  dispatch: Dispatch;
-};
-const ChatSetting: React.FC<Props> = () => {
-  return (
-    <>
-      <OverviewContainer mode={'chatSetting'} />
-    </>
-  );
-};
-
-export default connect(({ domainManger }: { domainManger: StateType }) => ({
-  domainManger,
-}))(ChatSetting);

webapp/packages/supersonic-fe/src/pages/SemanticModel/ChatSetting/ChatSettingSection.tsx

@@ -1,43 +0,0 @@
-import React, { useRef } from 'react';
-import { connect } from 'umi';
-import type { StateType } from '../model';
-import ProCard from '@ant-design/pro-card';
-import EntitySection from '../components/Entity/EntitySection';
-import { ChatConfigType } from '../enum';
-import type { Dispatch } from 'umi';
-
-type Props = {
-  domainManger: StateType;
-  dispatch: Dispatch;
-};
-
-const ChatSettingSection: React.FC<Props> = () => {
-  const metricRef = useRef<any>();
-  const tagRef = useRef<any>();
-  return (
-    <div style={{ width: 900, margin: '20px auto' }}>
-      <ProCard bordered title="指标模式" style={{ marginBottom: 20 }}>
-        <EntitySection
-          ref={metricRef}
-          chatConfigType={ChatConfigType.METRIC}
-          onConfigSave={() => {
-            tagRef.current.refreshConfigData();
-          }}
-        />
-      </ProCard>
-      <ProCard bordered title="标签模式" style={{ marginBottom: 20 }}>
-        <EntitySection
-          ref={tagRef}
-          chatConfigType={ChatConfigType.TAG}
-          onConfigSave={() => {
-            metricRef.current.refreshConfigData();
-          }}
-        />
-      </ProCard>
-    </div>
-  );
-};
-
-export default connect(({ domainManger }: { domainManger: StateType }) => ({
-  domainManger,
-}))(ChatSettingSection);

webapp/packages/supersonic-fe/src/pages/SemanticModel/ChatSetting/ChatSettingTab.tsx

@@ -1,102 +0,0 @@
-import { Tabs, Button } from 'antd';
-import React from 'react';
-import { connect } from 'umi';
-
-import styles from '../components/style.less';
-import type { StateType } from '../model';
-import { LeftOutlined } from '@ant-design/icons';
-import EntitySection from '../components/Entity/EntitySection';
-import RecommendedQuestionsSection from '../components/Entity/RecommendedQuestionsSection';
-import { ISemantic } from '../data';
-
-import OverView from '../components/OverView';
-import { ChatConfigType } from '../enum';
-import type { Dispatch } from 'umi';
-
-type Props = {
-  isModel: boolean;
-  activeKey: string;
-  modelList: ISemantic.IModelItem[];
-  handleModelChange: (model?: ISemantic.IModelItem) => void;
-  onBackDomainBtnClick?: () => void;
-  onMenuChange?: (menuKey: string) => void;
-  domainManger: StateType;
-  dispatch: Dispatch;
-};
-
-const ChatSetting: React.FC<Props> = ({
-  isModel,
-  activeKey,
-  modelList,
-  handleModelChange,
-  onBackDomainBtnClick,
-  onMenuChange,
-}) => {
-  const defaultTabKey = 'metric';
-
-  const isModelItem = [
-    {
-      label: '指标模式',
-      key: 'metric',
-      children: <EntitySection chatConfigType={ChatConfigType.METRIC} />,
-    },
-    {
-      label: '标签模式',
-      key: 'dimenstion',
-      children: <EntitySection chatConfigType={ChatConfigType.TAG} />,
-    },
-    {
-      label: '推荐问题',
-      key: 'recommendedQuestions',
-      children: <RecommendedQuestionsSection />,
-    },
-  ];
-
-  const tabItem = [
-    {
-      label: '模型',
-      key: 'overview',
-      children: (
-        <OverView
-          modelList={modelList}
-          disabledEdit={true}
-          onModelChange={(model) => {
-            handleModelChange(model);
-          }}
-        />
-      ),
-    },
-  ];
-
-  return (
-    <>
-      <Tabs
-        className={styles.tab}
-        items={isModel ? isModelItem : tabItem}
-        activeKey={activeKey || defaultTabKey}
-        destroyInactiveTabPane
-        tabBarExtraContent={
-          isModel ? (
-            <Button
-              type="primary"
-              icon={<LeftOutlined />}
-              onClick={() => {
-                onBackDomainBtnClick?.();
-              }}
-              style={{ marginRight: 10 }}
-            >
-              返回主题域
-            </Button>
-          ) : undefined
-        }
-        onChange={(menuKey: string) => {
-          onMenuChange?.(menuKey);
-        }}
-      />
-    </>
-  );
-};
-
-export default connect(({ domainManger }: { domainManger: StateType }) => ({
-  domainManger,
-}))(ChatSetting);

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/DataSourceCreateForm.tsx

@@ -18,6 +18,7 @@ export type CreateFormProps = {
   dispatch: Dispatch;
   createModalVisible: boolean;
   sql?: string;
+  sqlParams?: IDataSource.ISqlParamsItem[];
   databaseId?: number;
   modelItem: ISemantic.IModelItem;
   onCancel?: () => void;
@@ -42,6 +43,7 @@ const DataSourceCreateForm: React.FC<CreateFormProps> = ({
   createModalVisible,
   scriptColumns,
   sql = '',
+  sqlParams,
   onSubmit,
   modelItem,
   databaseId,
@@ -249,6 +251,7 @@ const DataSourceCreateForm: React.FC<CreateFormProps> = ({
           queryType: basicInfoFormMode === 'fast' ? 'table_query' : 'sql_query',
           tableQuery: dbName && tableName ? `${dbName}.${tableName}` : '',
           sqlQuery: sql,
+          sqlVariables: sqlParams,
         },
       };
       setQueryParamsState(queryParams);

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlDetail.tsx

@@ -12,17 +12,17 @@ import {
   SwapOutlined,
   PlayCircleOutlined,
   CloudServerOutlined,
+  ApiOutlined,
 } from '@ant-design/icons';
 import { isFunction } from 'lodash';
 import FullScreen from '@/components/FullScreen';
 import SqlEditor from '@/components/SqlEditor';
 import type { TaskResultItem, TaskResultColumn } from '../data';
 import { excuteSql } from '@/pages/SemanticModel/service';
-// import DataSourceCreateForm from './DataSourceCreateForm';
 import type { Dispatch } from 'umi';
 import type { StateType } from '../../model';
+import SqlParams from './SqlParams';
 import styles from '../style.less';
-
 import 'ace-builds/src-min-noconflict/ext-searchbox';
 import 'ace-builds/src-min-noconflict/theme-sqlserver';
 import 'ace-builds/src-min-noconflict/theme-monokai';
@@ -33,6 +33,7 @@ export type DataSourceSubmitData = {
   sql: string;
   databaseId: number;
   columns: any[];
+  sqlParams: any[];
 };
 
 type IProps = {
@@ -77,7 +78,6 @@ const SqlDetail: React.FC<IProps> = ({
   });
   const [dataBaseItems, setDataBaseItems] = useState<DatabaseItem[]>([]);
   const [currentDatabaseItem, setCurrentDatabaseItem] = useState<DatabaseItem>();
-  // const [dataSourceModalVisible, setDataSourceModalVisible] = useState(false);
 
   const [tableScroll, setTableScroll] = useState({
     scrollToFirstRowOnChange: true,
@@ -88,7 +88,7 @@ const SqlDetail: React.FC<IProps> = ({
   const [runState, setRunState] = useState<boolean | undefined>();
 
   const [taskLog, setTaskLog] = useState('');
-  const [isSqlExcLocked, setIsSqlExcLocked] = useState(false);
+  const [isSqlExcLocked, setIsSqlExcLocked] = useState<boolean>(false);
   const [screenSize, setScreenSize] = useState<ScreenSize>('middle');
 
   const [isSqlIdeFullScreen, setIsSqlIdeFullScreen] = useState<boolean>(false);
@@ -100,8 +100,11 @@ const SqlDetail: React.FC<IProps> = ({
   const DEFAULT_FULLSCREEN_TOP = 0;
 
   const [partialSql, setPartialSql] = useState('');
-  const [isPartial, setIsPartial] = useState(false);
-  const [isRight, setIsRight] = useState(false);
+  const [isPartial, setIsPartial] = useState<boolean>(false);
+  const [isRight, setIsRight] = useState<boolean>(false);
+
+  const [variableCollapsed, setVariableCollapsed] = useState<boolean>(true);
+  const [sqlParams, setSqlParams] = useState<IDataSource.ISqlParamsItem[]>([]);
 
   const [scriptColumns, setScriptColumns] = useState<any[]>([]);
 
@@ -125,6 +128,10 @@ const SqlDetail: React.FC<IProps> = ({
     setCurrentDatabaseItem(targetDataBase);
   }, [dataSourceItem, databaseConfigList]);
 
+  useEffect(() => {
+    setSqlParams(dataSourceItem?.modelDetail?.sqlVariables || []);
+  }, [dataSourceItem]);
+
   useEffect(() => {
     setRunState(undefined);
   }, [currentDatabaseItem, sql]);
@@ -138,6 +145,11 @@ const SqlDetail: React.FC<IProps> = ({
     return line;
   }
 
+  const handleVariable = () => {
+    const collapsedValue = !variableCollapsed;
+    setVariableCollapsed(collapsedValue);
+  };
+
   // 计算每列的宽度，通过容器插入文档中动态得到该列数据(包括表头)的最长宽度，设为列宽度，保证每列的数据都能一行展示完
   function getKeyWidthMap(list: TaskResultItem[]): TaskResultItem {
     const widthMap = {};
@@ -236,6 +248,7 @@ const SqlDetail: React.FC<IProps> = ({
     const { code, data, msg } = await excuteSql({
       sql: value,
       id: currentDatabaseItem.key,
+      sqlVariables: sqlParams,
     });
     setResultTableLoading(false);
     if (code === 200) {
@@ -417,6 +430,9 @@ const SqlDetail: React.FC<IProps> = ({
           <Tooltip title="格式化SQL语句">
             <EditOutlined className={styles.sqlOprIcon} onClick={formatSQL} />
           </Tooltip>
+          <Tooltip title="动态变量">
+            <ApiOutlined className={styles.sqlOprIcon} onClick={handleVariable} />
+          </Tooltip>
           <Tooltip title="改变主题">
             <SwapOutlined className={styles.sqlOprIcon} onClick={handleThemeChange} />
           </Tooltip>
@@ -463,6 +479,17 @@ const SqlDetail: React.FC<IProps> = ({
                 onSelect={onSelect}
               />
             </div>
+            <div
+              className={variableCollapsed ? styles.hideSqlParams : styles.sqlParams}
+              // style={{ height: sqlEditorHeight }}
+            >
+              <SqlParams
+                value={sqlParams}
+                onChange={(params) => {
+                  setSqlParams(params);
+                }}
+              />
+            </div>
           </div>
         </Pane>
         <div className={`${styles.sqlBottmWrap} ${screenSize}`}>
@@ -476,6 +503,7 @@ const SqlDetail: React.FC<IProps> = ({
                     columns: scriptColumns,
                     databaseId: currentDatabaseItem?.key || 0,
                     sql,
+                    sqlParams,
                   });
                 }}
                 disabled={!runState}

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlParams.tsx

@@ -0,0 +1,110 @@
+import { useState } from 'react';
+import type { FC } from 'react';
+import type { OprType } from '../data';
+import { IDataSource } from '../../data';
+import styles from '../style.less';
+import { AppstoreAddOutlined, DeleteTwoTone, EditTwoTone } from '@ant-design/icons';
+import SqlParamsDetailModal from './SqlParamsDetailModal';
+import { List } from 'antd';
+
+type Props = {
+  value?: IDataSource.ISqlParamsItem[];
+  onChange?: (e: IDataSource.ISqlParamsItem[]) => void;
+};
+
+const defalutItem: IDataSource.ISqlParamsItem = {
+  name: '',
+  defaultValues: [],
+  valueType: 'STRING',
+};
+
+const SqlParams: FC<Props> = ({ value, onChange }) => {
+  const [oprType, setOprType] = useState<OprType>('add');
+  const [visible, setVisible] = useState<boolean>(false);
+  const [initValue, setInitValue] = useState<IDataSource.ISqlParamsItem>();
+  const paramsChange = (params: IDataSource.ISqlParamsItem[]) => {
+    if (onChange) {
+      onChange(params);
+    }
+  };
+  const handleAdd = () => {
+    setOprType('add');
+    setVisible(true);
+    setInitValue(defalutItem);
+  };
+  const handleSave = async (values: IDataSource.ISqlParamsItem) => {
+    const newValue = value ? [...value] : [];
+    const { index, ...rest } = values;
+    if (index || index === 0) {
+      newValue[index] = rest;
+    } else {
+      newValue.push(rest);
+    }
+    setVisible(false);
+    setInitValue(undefined);
+    paramsChange(newValue);
+  };
+
+  const handleDelete = (index: number) => {
+    const newValue = value ? [...value] : [];
+    newValue.splice(index, 1);
+    paramsChange(newValue);
+  };
+  const handleEdit = (index: number) => {
+    const paramsItem = value ? value[index] : defalutItem;
+    setInitValue({ ...paramsItem, index });
+    setOprType('edit');
+    setVisible(true);
+  };
+  return (
+    <>
+      <div className={styles.sqlParamsBody}>
+        <div className={styles.header}>
+          <span className={styles.title}>变量</span>
+          <AppstoreAddOutlined className={styles.icon} onClick={handleAdd} />
+        </div>
+        <List
+          className={styles.paramsList}
+          dataSource={value}
+          renderItem={(item, index) => (
+            <List.Item
+              title={item.name}
+              className={styles.paramsItem}
+              key={item.name}
+              actions={[
+                <>
+                  <EditTwoTone
+                    className={styles.icon}
+                    onClick={() => {
+                      handleEdit(index);
+                    }}
+                  />
+                  <DeleteTwoTone
+                    className={styles.icon}
+                    onClick={() => {
+                      handleDelete(index);
+                    }}
+                  />
+                </>,
+              ]}
+            >
+              <div className={styles.name}>{item.name}</div>
+            </List.Item>
+          )}
+        />
+      </div>
+      <SqlParamsDetailModal
+        nameList={value?.map((item) => item.name)}
+        oprType={oprType}
+        modalVisible={visible}
+        onSave={handleSave}
+        onCancel={() => {
+          setVisible(false);
+        }}
+        initValue={initValue}
+      />
+    </>
+  );
+};
+
+export default SqlParams;

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlParamsDetailModal.tsx

@@ -0,0 +1,190 @@
+import { useEffect, useState } from 'react';
+import type { FC } from 'react';
+import { Modal, Form, Input, Select, Checkbox } from 'antd';
+import { isFunction } from 'lodash';
+import { objToArray } from '@/utils/utils';
+import type { ParamsItemProps, OprType } from '../data';
+import { IDataSource } from '../../data';
+import TextArea from 'antd/lib/input/TextArea';
+import ParamsSqlEditor from './SqlParamsSqlEditor';
+
+// const EnumSqlParamsType = {
+//   auth: '权限变量',
+//   query: '查询变量',
+// };
+
+const EnumSqlValueType = {
+  STRING: '字符串',
+  NUMBER: '数字',
+  EXPR: 'SQL表达式',
+};
+
+const { Option } = Select;
+
+const ParamsTextArea: FC<ParamsItemProps> = ({ value, onChange }) => {
+  return (
+    <TextArea
+      value={value && value[0]}
+      onChange={(e) => {
+        if (onChange) {
+          onChange([e.target.value]);
+        }
+      }}
+      placeholder="请输入表达式"
+      rows={3}
+    />
+  );
+};
+
+type IProps = {
+  oprType: OprType;
+  modalVisible: boolean;
+  onSave: (values: IDataSource.ISqlParamsItem) => Promise<any>;
+  onCancel?: (oprType: OprType) => void;
+  initValue?: IDataSource.ISqlParamsItem;
+  nameList?: string[];
+};
+
+const SqlParamsDetailModal: FC<IProps> = ({
+  oprType = 'add',
+  initValue = {} as IDataSource.ISqlParamsItem,
+  modalVisible,
+  onSave,
+  onCancel,
+  nameList,
+}) => {
+  const [valueType, setValueType] = useState<IDataSource.ISqlParamsValueType>();
+  const [oldName, setOldName] = useState<string>();
+
+  const formLayout = {
+    labelCol: { span: 7 },
+    wrapperCol: { span: 13 },
+  };
+
+  const [form] = Form.useForm();
+
+  const submitSave = async () => {
+    const fieldsValue = await form.validateFields();
+    onSave({ ...fieldsValue, index: initValue.index });
+  };
+
+  const handleCancel = async () => {
+    if (onCancel && isFunction(onCancel)) {
+      onCancel(oprType);
+    }
+  };
+
+  useEffect(() => {
+    form.setFieldsValue({
+      ...initValue,
+    });
+    setValueType(initValue.valueType);
+    setOldName(initValue.name);
+  }, [initValue]);
+
+  return (
+    <Modal
+      forceRender
+      title={`${oprType === 'add' ? '新建' : '编辑'}sql参数`}
+      open={modalVisible}
+      onOk={submitSave}
+      onCancel={handleCancel}
+    >
+      <Form
+        {...formLayout}
+        initialValues={{
+          ...initValue,
+        }}
+        form={form}
+      >
+        <Form.Item
+          name="name"
+          label="参数名称"
+          rules={[
+            { required: true, message: '请输入参数名称' },
+            {
+              validator(_, value, confirm) {
+                if (
+                  nameList?.some((item) => {
+                    return item === value && value !== oldName;
+                  })
+                ) {
+                  confirm('名称不能重复');
+                } else {
+                  confirm();
+                }
+              },
+            },
+          ]}
+        >
+          <Input placeholder="请输入参数名称" />
+        </Form.Item>
+        {/* <Form.Item
+          name="type"
+          label="参数类型"
+          rules={[{ required: true, message: '请选择参数类型' }]}
+        >
+          <Select placeholder="请选择" disabled={true}>
+            {objToArray(EnumSqlParamsType).map((d) => (
+              <Option key={d.value} value={d.value}>
+                {d.label}
+              </Option>
+            ))}
+          </Select>
+        </Form.Item> */}
+        <Form.Item
+          name="valueType"
+          label="值类型"
+          rules={[{ required: true, message: '请选择值类型' }]}
+        >
+          <Select
+            placeholder="请选择值类型"
+            onChange={(e) => {
+              setValueType(e as IDataSource.ISqlParamsValueType);
+              if (e === 'EXPR') {
+                form.setFieldsValue({
+                  defaultValues: undefined,
+                });
+              }
+            }}
+          >
+            {objToArray(EnumSqlValueType).map((d) => (
+              <Option key={d.value} value={d.value}>
+                {d.label}
+              </Option>
+            ))}
+          </Select>
+        </Form.Item>
+        {/* <Form.Item name="alias" label="别名">
+          <Input placeholder="请输入参数别名" />
+        </Form.Item> */}
+        {/* {valueType !== 'sql' && (
+          <Form.Item name="udf" label="是否使用表达式">
+            <Checkbox
+              checked={udf}
+              onChange={(e) => {
+                setUdf(e.target.checked);
+                if (e.target.checked) {
+                  form.setFieldsValue({
+                    defaultValues: '',
+                  });
+                }
+              }}
+            />
+          </Form.Item>
+        )} */}
+        {valueType === 'EXPR' ? (
+          <Form.Item name="defaultValues" label="表达式">
+            <ParamsTextArea />
+          </Form.Item>
+        ) : (
+          <Form.Item name="defaultValues" label="默认值">
+            <ParamsSqlEditor />
+          </Form.Item>
+        )}
+      </Form>
+    </Modal>
+  );
+};
+
+export default SqlParamsDetailModal;

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/SqlParamsSqlEditor.tsx

@@ -0,0 +1,132 @@
+import React, { useState, useRef } from 'react';
+import type { FC } from 'react';
+import { Input, Tag, Tooltip } from 'antd';
+import { PlusOutlined } from '@ant-design/icons';
+
+import styles from '../style.less';
+
+type ParamsItemProps = {
+  value?: string[];
+  onChange?: (e: string[]) => void;
+};
+const ParamsSqlEditor: FC<ParamsItemProps> = ({ value, onChange }) => {
+  const [editInputValue, setEditInputValue] = useState<string>();
+  const [inputValue, setInputValue] = useState<string>();
+  const [editInputIndex, setEditInputIndex] = useState<number>(-1);
+  const [inputVisible, setInputVisible] = useState<boolean>(false);
+  const editInput = useRef<typeof Input>(null);
+  const inputRef = useRef<typeof Input>(null);
+
+  const handleEditInputChange: React.ChangeEventHandler<HTMLInputElement> = (e) => {
+    setEditInputValue(e.target.value);
+  };
+  const handleEditInputConfirm = () => {
+    const newValues = value ? [...value] : [];
+    newValues[editInputIndex] = editInputValue || '';
+    if (onChange) {
+      onChange(newValues);
+    }
+    setEditInputIndex(-1);
+    setEditInputValue('');
+  };
+  const handleClose = (removedTag: string) => {
+    const newValues = value ? value.filter((tag) => tag !== removedTag) : [];
+    if (onChange) {
+      onChange(newValues);
+    }
+  };
+  const handleInputChange: React.ChangeEventHandler<HTMLInputElement> = (e) => {
+    setInputValue(e.target.value);
+  };
+
+  const handleInputConfirm = () => {
+    const newValues = value ? [...value] : [];
+    if (inputValue && !newValues.includes(inputValue)) {
+      newValues.push(inputValue);
+    }
+    if (onChange) {
+      onChange(newValues);
+    }
+    setInputVisible(false);
+    setInputValue('');
+  };
+
+  const showInput = () => {
+    setInputVisible(true);
+    setTimeout(() => {
+      inputRef.current?.focus();
+    }, 0);
+  };
+
+  return (
+    <>
+      {value &&
+        value.map((tag: any, index: number) => {
+          if (editInputIndex === index) {
+            return (
+              <Input
+                ref={editInput}
+                key={tag}
+                size="small"
+                className={styles.tagInput}
+                value={editInputValue}
+                onChange={handleEditInputChange}
+                onBlur={handleEditInputConfirm}
+                onPressEnter={handleEditInputConfirm}
+              />
+            );
+          }
+
+          const isLongTag = tag.length > 20;
+
+          const tagElem = (
+            <Tag
+              className={styles.editTag}
+              key={tag}
+              closable={true}
+              onClose={() => handleClose(tag)}
+            >
+              <span
+                onDoubleClick={(e) => {
+                  setEditInputIndex(index);
+                  setEditInputValue(tag);
+                  e.preventDefault();
+                  setTimeout(() => {
+                    editInput.current?.focus();
+                  }, 0);
+                }}
+              >
+                {isLongTag ? `${tag.slice(0, 20)}...` : tag}
+              </span>
+            </Tag>
+          );
+          return isLongTag ? (
+            <Tooltip title={tag} key={tag}>
+              {tagElem}
+            </Tooltip>
+          ) : (
+            tagElem
+          );
+        })}
+      {inputVisible && (
+        <Input
+          ref={inputRef}
+          type="text"
+          size="small"
+          className={styles.tagInput}
+          value={inputValue}
+          onChange={handleInputChange}
+          onBlur={handleInputConfirm}
+          onPressEnter={handleInputConfirm}
+        />
+      )}
+      {!inputVisible && (
+        <Tag className={styles.siteTagPlus} onClick={showInput}>
+          <PlusOutlined /> 增加默认值
+        </Tag>
+      )}
+    </>
+  );
+};
+
+export default ParamsSqlEditor;

webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/data.d.ts

@@ -6,3 +6,10 @@ export type TaskResultColumn = {
 
 // 任务查询结果
 export type TaskResultItem = Record<string, string | number>;
+
+export type OprType = 'add' | 'edit';
+
+export type ParamsItemProps = {
+  value?: any;
+  onChange?: (e: any) => void;
+};

webapp/packages/supersonic-fe/src/pages/SemanticModel/OverviewContainer.tsx

@@ -233,39 +233,21 @@ const OverviewContainer: React.FC<Props> = ({ mode, domainManger, dispatch }) =>
         <div className={styles.content}>
           {selectDomainId ? (
             <>
-              {mode === 'domain' ? (
-                <DomainManagerTab
-                  isModel={isModel}
-                  activeKey={activeKey}
-                  modelList={modelList}
-                  handleModelChange={(model) => {
-                    handleModelChange(model);
-                  }}
-                  onBackDomainBtnClick={() => {
-                    cleanModelInfo(selectDomainId);
-                  }}
-                  onMenuChange={(menuKey) => {
-                    setActiveKey(menuKey);
-                    pushUrlMenu(selectDomainId, selectModelId, menuKey);
-                  }}
-                />
-              ) : (
-                <ChatSettingTab
-                  isModel={isModel}
-                  activeKey={activeKey}
-                  modelList={modelList}
-                  handleModelChange={(model) => {
-                    handleModelChange(model);
-                  }}
-                  onBackDomainBtnClick={() => {
-                    cleanModelInfo(selectDomainId);
-                  }}
-                  onMenuChange={(menuKey) => {
-                    setActiveKey(menuKey);
-                    pushUrlMenu(selectDomainId, selectModelId, menuKey);
-                  }}
-                />
-              )}
+              <DomainManagerTab
+                isModel={isModel}
+                activeKey={activeKey}
+                modelList={modelList}
+                handleModelChange={(model) => {
+                  handleModelChange(model);
+                }}
+                onBackDomainBtnClick={() => {
+                  cleanModelInfo(selectDomainId);
+                }}
+                onMenuChange={(menuKey) => {
+                  setActiveKey(menuKey);
+                  pushUrlMenu(selectDomainId, selectModelId, menuKey);
+                }}
+              />
             </>
           ) : (
             <h2 className={styles.mainTip}>请选择项目</h2>

webapp/packages/supersonic-fe/src/pages/SemanticModel/View/components/ViewCreateFormModal.tsx

@@ -7,13 +7,8 @@ import { createView, updateView, getDimensionList, queryMetric } from '../../ser
 import { ISemantic } from '../../data';
 import { isString } from 'lodash';
 import FormItemTitle from '@/components/FormHelper/FormItemTitle';
-import SelectPartner from '@/components/SelectPartner';
 import SelectTMEPerson from '@/components/SelectTMEPerson';
 import ViewModelConfigTransfer from './ViewModelConfigTransfer';
-import DefaultSettingForm from './DefaultSettingForm';
-import SqlEditor from '@/components/SqlEditor';
-import ProCard from '@ant-design/pro-card';
-import { ChatConfigType } from '../../enum';
 
 const FormItem = Form.Item;
 

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/ClassDataSourceTypeModal.tsx

@@ -41,6 +41,7 @@ const ClassDataSourceTypeModal: React.FC<Props> = ({
   const [dataSourceEditOpen, setDataSourceEditOpen] = useState<boolean>(false);
   const [currentDatabaseId, setCurrentDatabaseId] = useState<number>();
   const [scriptColumns, setScriptColumns] = useState<IDataSource.IExecuteSqlColumn[]>([]);
+  const [sqlParams, setSqlParams] = useState<IDataSource.ISqlParamsItem[]>([]);
 
   useEffect(() => {
     if (!dataSourceItem?.id || !open) {
@@ -88,23 +89,6 @@ const ClassDataSourceTypeModal: React.FC<Props> = ({
     }
   }, [dataSourceItem]);
 
-  // const fetchTaskResult = (params) => {
-  //   setScriptColumns(params.columns);
-  // };
-
-  // const queryTableColumnListByScript = async (dataSource: IDataSource.IDataSourceItem) => {
-  //   if (!dataSource?.modelDetail?.sqlQuery) {
-  //     return;
-  //   }
-  //   const { code, data } = await excuteSql({
-  //     sql: dataSource.modelDetail?.sqlQuery,
-  //     id: dataSource.databaseId,
-  //   });
-  //   if (code === 200) {
-  //     fetchTaskResult(data);
-  //   }
-  // };
-
   return (
     <>
       <Modal
@@ -187,6 +171,7 @@ const ClassDataSourceTypeModal: React.FC<Props> = ({
           basicInfoFormMode="normal"
           modelItem={dataSourceItem}
           scriptColumns={scriptColumns}
+          sqlParams={sqlParams}
           onCancel={() => {
             setCreateModalVisible(false);
             handleCancel();
@@ -215,9 +200,11 @@ const ClassDataSourceTypeModal: React.FC<Props> = ({
             <DataSource
               initialValues={dataSourceItem}
               onSubmitSuccess={(dataSourceInfo) => {
-                const { columns, sql, databaseId } = dataSourceInfo;
+                console.log('onSubmitSuccess', dataSourceInfo);
+                const { columns, sql, databaseId, sqlParams } = dataSourceInfo;
                 setSql(sql);
                 setScriptColumns(columns);
+                setSqlParams(sqlParams);
                 setCurrentDatabaseId(databaseId);
                 setDataSourceEditOpen(false);
               }}

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/DimensionValueSettingModal.tsx

@@ -184,7 +184,7 @@ const DimensionValueSettingModal: React.FC<CreateFormProps> = ({
     {
       label: '维度值设置',
       key: 'setting',
-      children: <DimensionValueSettingForm modelId={modelId} dimensionItem={dimensionItem} />,
+      children: <DimensionValueSettingForm dimensionItem={dimensionItem} />,
     },
   ];
 

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/Entity/DefaultSettingForm.tsx

@@ -1,244 +0,0 @@
-import { useEffect, useState, forwardRef, useImperativeHandle } from 'react';
-import type { ForwardRefRenderFunction } from 'react';
-import FormItemTitle from '@/components/FormHelper/FormItemTitle';
-import { formLayout } from '@/components/FormHelper/utils';
-import { message, Form, Input, Select, Button, InputNumber } from 'antd';
-import { addDomainExtend, editDomainExtend } from '../../service';
-import {
-  formatRichEntityDataListToIds,
-  wrapperTransTypeAndId,
-  splitListToTransTypeId,
-} from './utils';
-import styles from '../style.less';
-import { ISemantic } from '../../data';
-import { ChatConfigType, TransType, SemanticNodeType } from '../../enum';
-import TransTypeTag from '../TransTypeTag';
-
-type Props = {
-  entityData: any;
-  chatConfigKey: string;
-  chatConfigType: ChatConfigType.TAG | ChatConfigType.METRIC;
-  metricList: ISemantic.IMetricItem[];
-  dimensionList: ISemantic.IDimensionItem[];
-  domainId: number;
-  onSubmit: (params?: any) => void;
-};
-
-const FormItem = Form.Item;
-const Option = Select.Option;
-
-const formDefaultValue = {
-  unit: 7,
-  period: 'DAY',
-  timeMode: 'LAST',
-};
-
-const DefaultSettingForm: ForwardRefRenderFunction<any, Props> = (
-  { metricList, dimensionList, entityData, chatConfigKey, chatConfigType, onSubmit },
-  ref,
-) => {
-  const [form] = Form.useForm();
-  const [dataItemListOptions, setDataItemListOptions] = useState<any>([]);
-  const formatEntityData = formatRichEntityDataListToIds(entityData);
-  const getFormValidateFields = async () => {
-    return await form.validateFields();
-  };
-
-  useImperativeHandle(ref, () => ({
-    getFormValidateFields,
-  }));
-
-  useEffect(() => {
-    form.resetFields();
-    if (!(entityData?.id && entityData?.chatDefaultConfig)) {
-      return;
-    }
-    const { chatDefaultConfig, id } = formatEntityData;
-    form.setFieldsValue({
-      ...formDefaultValue,
-      ...chatDefaultConfig,
-      id,
-    });
-    if (chatConfigType === ChatConfigType.TAG) {
-      initDataItemValue(chatDefaultConfig);
-    }
-  }, [entityData, dataItemListOptions]);
-
-  const initDataItemValue = (chatDefaultConfig: {
-    dimensionIds: number[];
-    metricIds: number[];
-  }) => {
-    const { dimensionIds, metricIds } = chatDefaultConfig;
-    const dimensionIdString = dimensionIds.map((dimensionId: number) => {
-      return wrapperTransTypeAndId(TransType.DIMENSION, dimensionId);
-    });
-    const metricIdString = metricIds.map((metricId: number) => {
-      return wrapperTransTypeAndId(TransType.METRIC, metricId);
-    });
-    form.setFieldsValue({
-      dataItemIds: [...dimensionIdString, ...metricIdString],
-    });
-  };
-
-  useEffect(() => {
-    if (Array.isArray(dimensionList) && Array.isArray(metricList)) {
-      const dimensionEnum = dimensionList.map((item: ISemantic.IDimensionItem) => {
-        const { name, id, bizName } = item;
-        return {
-          name,
-          label: (
-            <>
-              <TransTypeTag type={SemanticNodeType.DIMENSION} />
-              {name}
-            </>
-          ),
-          value: wrapperTransTypeAndId(TransType.DIMENSION, id),
-          bizName,
-          id,
-          transType: TransType.DIMENSION,
-        };
-      });
-      const metricEnum = metricList.map((item: ISemantic.IMetricItem) => {
-        const { name, id, bizName } = item;
-        return {
-          name,
-          label: (
-            <>
-              <TransTypeTag type={SemanticNodeType.METRIC} />
-              {name}
-            </>
-          ),
-          value: wrapperTransTypeAndId(TransType.METRIC, id),
-          bizName,
-          id,
-          transType: TransType.METRIC,
-        };
-      });
-      setDataItemListOptions([...dimensionEnum, ...metricEnum]);
-    }
-  }, [dimensionList, metricList]);
-
-  const saveEntity = async () => {
-    const values = await form.validateFields();
-    const { id, dataItemIds } = values;
-    let dimensionConfig = {};
-    if (dataItemIds) {
-      const { dimensionIds, metricIds } = splitListToTransTypeId(dataItemIds);
-      dimensionConfig = {
-        dimensionIds,
-        metricIds,
-      };
-    }
-    let saveDomainExtendQuery = addDomainExtend;
-    if (id) {
-      saveDomainExtendQuery = editDomainExtend;
-    }
-    const params = {
-      ...formatEntityData,
-      chatDefaultConfig: { ...values, ...dimensionConfig },
-    };
-    const { modelId } = entityData;
-    const { code, msg, data } = await saveDomainExtendQuery({
-      [chatConfigKey]: params,
-      modelId,
-      id,
-    });
-    if (code === 200) {
-      form.setFieldValue('id', data);
-      onSubmit?.();
-      message.success('保存成功');
-      return;
-    }
-    message.error(msg);
-  };
-
-  return (
-    <>
-      <Form
-        {...formLayout}
-        form={form}
-        layout="vertical"
-        className={styles.form}
-        initialValues={formDefaultValue}
-      >
-        <FormItem hidden={true} name="id" label="ID">
-          <Input placeholder="id" />
-        </FormItem>
-
-        {chatConfigType === ChatConfigType.TAG && (
-          <FormItem name="dataItemIds" label="圈选结果展示字段">
-            <Select
-              mode="multiple"
-              allowClear
-              style={{ width: '100%' }}
-              optionLabelProp="name"
-              filterOption={(inputValue: string, item: any) => {
-                const { name } = item;
-                if (name.includes(inputValue)) {
-                  return true;
-                }
-                return false;
-              }}
-              placeholder="请选择圈选结果展示字段"
-              options={dataItemListOptions}
-            />
-          </FormItem>
-        )}
-        <FormItem
-          label={
-            <FormItemTitle
-              title={'时间范围'}
-              subTitle={'问答搜索结果选择中，如果没有指定时间范围，将会采用默认时间范围'}
-            />
-          }
-        >
-          <Input.Group compact>
-            {chatConfigType === ChatConfigType.TAG ? (
-              <span
-                style={{
-                  display: 'inline-block',
-                  lineHeight: '32px',
-                  marginRight: '8px',
-                }}
-              >
-                前
-              </span>
-            ) : (
-              <>
-                <FormItem name={'timeMode'} noStyle>
-                  <Select style={{ width: '90px' }}>
-                    <Option value="LAST">前</Option>
-                    <Option value="RECENT">最近</Option>
-                  </Select>
-                </FormItem>
-              </>
-            )}
-            <FormItem name={'unit'} noStyle>
-              <InputNumber style={{ width: '120px' }} />
-            </FormItem>
-            <FormItem name={'period'} noStyle>
-              <Select style={{ width: '90px' }}>
-                <Option value="DAY">天</Option>
-                <Option value="WEEK">周</Option>
-                <Option value="MONTH">月</Option>
-                <Option value="YEAR">年</Option>
-              </Select>
-            </FormItem>
-          </Input.Group>
-        </FormItem>
-        <FormItem>
-          <Button
-            type="primary"
-            onClick={() => {
-              saveEntity();
-            }}
-          >
-            保 存
-          </Button>
-        </FormItem>
-      </Form>
-    </>
-  );
-};
-
-export default forwardRef(DefaultSettingForm);

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/Entity/DimensionValueSettingForm.tsx

@@ -1,107 +1,118 @@
 import { useState, useEffect, forwardRef, useImperativeHandle } from 'react';
 import type { ForwardRefRenderFunction } from 'react';
-import { Form, Input, Switch, Space, Button, Divider, Tooltip, message } from 'antd';
+import { Form, Switch, Space, Button, Tooltip, message, Select } from 'antd';
 import FormItemTitle from '@/components/FormHelper/FormItemTitle';
 import { RedoOutlined, InfoCircleOutlined } from '@ant-design/icons';
 import { formLayout } from '@/components/FormHelper/utils';
-import { DictTaskState, TransType } from '../../enum';
+import { DictTaskState, KnowledgeConfigTypeEnum, KnowledgeConfigStatusEnum } from '../../enum';
 import {
-  getDomainExtendDetailConfig,
-  addDomainExtend,
-  editDomainExtend,
   searchKnowledgeConfigQuery,
   searchDictLatestTaskList,
   createDictTask,
+  editDictConfig,
+  createDictConfig,
+  deleteDictTask,
 } from '../../service';
-import type { IChatConfig, ISemantic } from '../../data';
+import type { ISemantic } from '../../data';
 import { isString } from 'lodash';
 import styles from '../style.less';
 import CommonEditList from '../../components/CommonEditList';
 
 type Props = {
-  modelId: number;
   dimensionItem: ISemantic.IDimensionItem;
   onSubmit?: () => void;
 };
 
-type TaskStateMap = Record<string, DictTaskState>;
-
 const FormItem = Form.Item;
 
 const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
-  { modelId, dimensionItem },
+  { dimensionItem },
   ref,
 ) => {
   const [form] = Form.useForm();
 
   const exchangeFields = ['blackList', 'whiteList'];
-  const [modelRichConfigData, setModelRichConfigData] = useState<IChatConfig.IConfig>();
   const [dimensionVisible, setDimensionVisible] = useState<boolean>(false);
-  const [taskStateMap, setTaskStateMap] = useState<TaskStateMap>({});
+  const [taskItemState, setTaskItemState] = useState<ISemantic.IDictKnowledgeTaskItem>();
   const [saveLoading, setSaveLoading] = useState<boolean>(false);
   const [refreshLoading, setRefreshLoading] = useState<boolean>(false);
+  const [knowledgeConfig, setKnowledgeConfig] = useState<ISemantic.IDictKnowledgeConfigItem>();
 
-  const queryThemeListData: any = async () => {
-    const { code, data } = await getDomainExtendDetailConfig({
-      modelId,
-    });
+  const [deleteLoading, setDeleteLoading] = useState<boolean>(false);
+  const [importDictState, setImportDictState] = useState<boolean>(false);
 
-    if (code === 200) {
-      setModelRichConfigData(data);
-      const targetKnowledgeInfos = data?.chatAggRichConfig?.knowledgeInfos || [];
-      const targetConfig = targetKnowledgeInfos.find(
-        (item: IChatConfig.IKnowledgeInfosItem) => item.itemId === dimensionItem.id,
-      );
-      if (targetConfig) {
-        const { knowledgeAdvancedConfig, searchEnable } = targetConfig;
-        setDimensionVisible(searchEnable);
-        const { blackList, whiteList, ruleList } = knowledgeAdvancedConfig;
-        form.setFieldsValue({
-          blackList: blackList.join(','),
-          whiteList: whiteList.join(','),
-          ruleList: ruleList || [],
-        });
-      }
-      return;
-    }
-
-    message.error('获取问答设置信息失败');
+  const defaultKnowledgeConfig: ISemantic.IDictKnowledgeConfigItemConfig = {
+    blackList: [],
+    whiteList: [],
+    ruleList: [],
   };
 
   useEffect(() => {
-    queryThemeListData();
+    searchKnowledgeConfig();
     queryDictLatestTaskList();
   }, []);
 
-  const taskRender = (dimension: ISemantic.IDimensionItem) => {
-    const { id, type } = dimension;
-    const target = taskStateMap[id];
-    if (type === TransType.DIMENSION && target) {
-      return DictTaskState[target] || '未知状态';
+  const taskRender = () => {
+    if (taskItemState?.taskStatus) {
+      return (
+        <span style={{ color: '#5493ff', fontWeight: 'bold' }}>
+          {DictTaskState[taskItemState.taskStatus] || '未知状态'}
+        </span>
+      );
     }
     return '--';
   };
 
+  const searchKnowledgeConfig = async () => {
+    setRefreshLoading(true);
+    const { code, data } = await searchKnowledgeConfigQuery({
+      type: KnowledgeConfigTypeEnum.DIMENSION,
+      itemId: dimensionItem.id,
+    });
+
+    setRefreshLoading(false);
+    if (code !== 200) {
+      message.error('获取字典导入配置失败!');
+      return;
+    }
+    const configItem = data[0];
+    if (configItem) {
+      const { status, config } = configItem;
+      if (status === KnowledgeConfigStatusEnum.ONLINE) {
+        setDimensionVisible(true);
+      } else {
+        setDimensionVisible(false);
+      }
+      form.setFieldsValue({
+        ...config,
+      });
+      setKnowledgeConfig(configItem);
+    } else {
+      form.setFieldsValue({
+        ...defaultKnowledgeConfig,
+      });
+      createDictConfigQuery(dimensionItem, defaultKnowledgeConfig);
+    }
+  };
+
   const queryDictLatestTaskList = async () => {
     setRefreshLoading(true);
-    searchKnowledgeConfigQuery({ itemId: dimensionItem.id });
     const { code, data } = await searchDictLatestTaskList({
-      modelId,
+      type: KnowledgeConfigTypeEnum.DIMENSION,
+      itemId: dimensionItem.id,
     });
     setRefreshLoading(false);
     if (code !== 200) {
       message.error('获取字典导入任务失败!');
       return;
     }
-    const tastMap = data.reduce(
-      (stateMap: TaskStateMap, item: { dimId: number; status: DictTaskState }) => {
-        const { dimId, status } = item;
-        stateMap[dimId] = status;
-        return stateMap;
-      },
-      {},
-    );
-    setTaskStateMap(tastMap);
+
+    if (data?.id) {
+      if (data.taskStatus !== 'running') {
+        setImportDictState(false);
+      }
+      setTaskItemState(data);
+    }
   };
 
   const getFormValidateFields = async () => {
@@ -128,12 +139,29 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
     getFormValidateFields,
   }));
 
+  const createDictConfigQuery = async (
+    dimension: ISemantic.IDimensionItem,
+    config: ISemantic.IDictKnowledgeConfigItemConfig,
+  ) => {
+    const { code, data } = await createDictConfig({
+      type: KnowledgeConfigTypeEnum.DIMENSION,
+      itemId: dimension.id,
+      config,
+      status: 1,
+    });
+
+    if (code !== 200) {
+      message.error('字典导入配置创建失败!');
+      return;
+    }
+    setKnowledgeConfig(data);
+  };
+
   const createDictTaskQuery = async (dimension: ISemantic.IDimensionItem) => {
+    setImportDictState(true);
     const { code } = await createDictTask({
-      updateMode: 'REALTIME_ADD',
-      modelAndDimPair: {
-        [modelId]: [dimension.id],
-      },
+      type: KnowledgeConfigTypeEnum.DIMENSION,
+      itemId: dimension.id,
     });
 
     if (code !== 200) {
@@ -145,73 +173,40 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
     }, 2000);
   };
 
-  const saveEntity = async (searchEnable = dimensionVisible) => {
-    setSaveLoading(true);
-    const globalKnowledgeConfigFormFields: any = await getFormValidateFields();
-
-    const tempData = { ...modelRichConfigData };
-    const targetKnowledgeInfos = modelRichConfigData?.chatAggRichConfig?.knowledgeInfos || [];
-
-    const hasHistoryConfig = targetKnowledgeInfos.find((item) => item.itemId === dimensionItem.id);
-    let knowledgeInfos: IChatConfig.IKnowledgeInfosItem[] = targetKnowledgeInfos;
-
-    if (hasHistoryConfig) {
-      knowledgeInfos = targetKnowledgeInfos.reduce(
-        (
-          knowledgeInfosList: IChatConfig.IKnowledgeInfosItem[],
-          item: IChatConfig.IKnowledgeInfosItem,
-        ) => {
-          if (item.itemId === dimensionItem.id) {
-            knowledgeInfosList.push({
-              ...item,
-              knowledgeAdvancedConfig: {
-                ...item.knowledgeAdvancedConfig,
-                ...globalKnowledgeConfigFormFields,
-              },
-              searchEnable,
-            });
-          } else {
-            knowledgeInfosList.push({
-              ...item,
-            });
-          }
-          return knowledgeInfosList;
-        },
-        [],
-      );
-    } else {
-      knowledgeInfos.push({
-        itemId: dimensionItem.id,
-        bizName: dimensionItem.bizName,
-        knowledgeAdvancedConfig: {
-          ...globalKnowledgeConfigFormFields,
-        },
-        searchEnable,
-      });
-    }
-
-    const { id, modelId, chatAggRichConfig } = tempData;
-    const saveParams = {
-      id,
-      modelId,
-      chatAggConfig: {
-        ...chatAggRichConfig,
-        knowledgeInfos,
-      },
-    };
-    let saveDomainExtendQuery = addDomainExtend;
-    if (id) {
-      saveDomainExtendQuery = editDomainExtend;
-    }
-
-    const { code, msg } = await saveDomainExtendQuery({
-      ...saveParams,
-    });
-    setSaveLoading(false);
-    if (code === 200) {
+  const editDictTaskQuery = async (
+    status: KnowledgeConfigStatusEnum = KnowledgeConfigStatusEnum.ONLINE,
+  ) => {
+    if (!knowledgeConfig?.id) {
+      return;
+    }
+    const config = await form.validateFields();
+    setSaveLoading(true);
+    const { code } = await editDictConfig({
+      ...knowledgeConfig,
+      config: {
+        ...knowledgeConfig.config,
+        ...config,
+      },
+      status,
+    });
+    setSaveLoading(false);
+    if (code !== 200) {
+      message.error('字典导入配置保存失败!');
+      return;
+    }
+  };
+
+  const deleteDictTaskQuery = async (dimension: ISemantic.IDimensionItem) => {
+    setDeleteLoading(true);
+    const { code } = await deleteDictTask({
+      type: KnowledgeConfigTypeEnum.DIMENSION,
+      itemId: dimension.id,
+    });
+    setDeleteLoading(false);
+    if (code !== 200) {
+      message.error('字典清除失败!');
       return;
     }
-    message.error(msg);
   };
 
   return (
@@ -236,7 +231,11 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
                       size="small"
                       checked={dimensionVisible}
                       onChange={(value) => {
-                        saveEntity(value);
+                        editDictTaskQuery(
+                          value
+                            ? KnowledgeConfigStatusEnum.ONLINE
+                            : KnowledgeConfigStatusEnum.OFFLINE,
+                        );
                         setDimensionVisible(value);
                       }}
                     />
@@ -249,21 +248,27 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
         >
           {dimensionVisible && (
             <Space size={20} style={{ marginBottom: 20 }}>
-              <Button
-                type="link"
-                size="small"
-                style={{ padding: 0 }}
-                onClick={(event) => {
-                  createDictTaskQuery(dimensionItem);
-                  event.stopPropagation();
-                }}
-              >
-                <Tooltip title="立即将维度值导入字典">
+              <Tooltip title="立即将维度值导入字典">
+                <Button
+                  type="link"
+                  size="small"
+                  style={{ padding: 0 }}
+                  disabled={importDictState}
+                  onClick={(event) => {
+                    createDictTaskQuery(dimensionItem);
+                    setTaskItemState({
+                      ...(taskItemState || ({} as ISemantic.IDictKnowledgeTaskItem)),
+                      taskStatus: 'running',
+                    });
+
+                    event.stopPropagation();
+                  }}
+                >
                   <Space>
                     立即导入字典 <InfoCircleOutlined />
                   </Space>
-                </Tooltip>
-              </Button>
+                </Button>
+              </Tooltip>
 
               <Tooltip title="刷新字典任务状态">
                 <Space>
@@ -276,12 +281,34 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
                     }}
                   >
                     导入状态
-                    <RedoOutlined />:
+                    <Space>
+                      <RedoOutlined />: <span>{taskRender(dimensionItem)}</span>
+                    </Space>
                   </Button>
                 </Space>
               </Tooltip>
 
-              <span>{taskRender(dimensionItem)}</span>
+              <Button
+                type="link"
+                size="small"
+                style={{ padding: 0 }}
+                disabled={taskItemState?.taskStatus === 'running'}
+                loading={deleteLoading}
+                onClick={(event) => {
+                  deleteDictTaskQuery(dimensionItem);
+                  setTaskItemState({
+                    ...(taskItemState || ({} as ISemantic.IDictKnowledgeTaskItem)),
+                    taskStatus: '',
+                  });
+                  event.stopPropagation();
+                }}
+              >
+                <Tooltip title="清除当前配置的字典">
+                  <Space>
+                    清除字典 <InfoCircleOutlined />
+                  </Space>
+                </Tooltip>
+              </Button>
             </Space>
           )}
         </FormItem>
@@ -306,7 +333,7 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
                   <Button
                     type="primary"
                     onClick={() => {
-                      saveEntity();
+                      editDictTaskQuery();
                     }}
                     loading={saveLoading}
                   >
@@ -316,11 +343,21 @@ const DimensionValueSettingForm: ForwardRefRenderFunction<any, Props> = (
               </div>
 
               <FormItem name="blackList" label="黑名单">
-                <Input placeholder="多个维度值用英文逗号隔开" />
+                <Select
+                  mode="tags"
+                  placeholder="输入维度值后回车确认，多别名输入、复制粘贴支持英文逗号自动分隔"
+                  tokenSeparators={[',']}
+                  maxTagCount={9}
+                />
               </FormItem>
 
               <FormItem name="whiteList" label="白名单">
-                <Input placeholder="多个维度值用英文逗号隔开" />
+                <Select
+                  mode="tags"
+                  placeholder="输入维度值后回车确认，多别名输入、复制粘贴支持英文逗号自动分隔"
+                  tokenSeparators={[',']}
+                  maxTagCount={9}
+                />
               </FormItem>
 
               <FormItem name="ruleList">

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/Entity/EntitySection.tsx

@@ -1,103 +0,0 @@
-import { message, Space } from 'antd';
-import { useEffect, useState, forwardRef, useImperativeHandle, Ref } from 'react';
-import type { Dispatch } from 'umi';
-import { connect } from 'umi';
-import type { StateType } from '../../model';
-import { getDomainExtendDetailConfig } from '../../service';
-import ProCard from '@ant-design/pro-card';
-
-import DefaultSettingForm from './DefaultSettingForm';
-import type { IChatConfig } from '../../data';
-import { ChatConfigType } from '../../enum';
-
-type Props = {
-  chatConfigType: ChatConfigType.TAG | ChatConfigType.METRIC;
-  onConfigSave?: () => void;
-  dispatch: Dispatch;
-  domainManger: StateType;
-};
-
-const EntitySection: React.FC<Props> = forwardRef(
-  ({ domainManger, chatConfigType = ChatConfigType.TAG, onConfigSave }, ref: Ref<any>) => {
-    const { selectDomainId, selectModelId: modelId, dimensionList, metricList } = domainManger;
-
-    const [entityData, setEntityData] = useState<IChatConfig.IChatRichConfig>();
-
-    useImperativeHandle(ref, () => ({
-      refreshConfigData: queryThemeListData,
-    }));
-
-    const queryThemeListData: any = async () => {
-      const { code, data } = await getDomainExtendDetailConfig({
-        modelId,
-      });
-
-      if (code === 200) {
-        const { chatAggRichConfig, chatDetailRichConfig, id, domainId, modelId } = data;
-        if (chatConfigType === ChatConfigType.TAG) {
-          setEntityData({ ...chatDetailRichConfig, id, domainId, modelId });
-        }
-        if (chatConfigType === ChatConfigType.METRIC) {
-          setEntityData({ ...chatAggRichConfig, id, domainId, modelId });
-        }
-        return;
-      }
-
-      message.error('获取问答设置信息失败');
-    };
-
-    const initPage = async () => {
-      queryThemeListData();
-    };
-
-    useEffect(() => {
-      if (!modelId) {
-        return;
-      }
-      initPage();
-    }, [modelId]);
-
-    return (
-      <div style={{ width: 800, margin: '0 auto' }}>
-        <Space direction="vertical" style={{ width: '100%' }} size={20}>
-          <ProCard bordered title="默认设置">
-            <DefaultSettingForm
-              domainId={Number(selectDomainId)}
-              entityData={entityData || {}}
-              chatConfigType={chatConfigType}
-              chatConfigKey={
-                chatConfigType === ChatConfigType.TAG ? 'chatDetailConfig' : 'chatAggConfig'
-              }
-              dimensionList={dimensionList.filter((item) => {
-                const blackDimensionList = entityData?.visibility?.blackDimIdList;
-                if (Array.isArray(blackDimensionList)) {
-                  return !blackDimensionList.includes(item.id);
-                }
-                return false;
-              })}
-              metricList={metricList.filter((item) => {
-                const blackMetricIdList = entityData?.visibility?.blackMetricIdList;
-                if (Array.isArray(blackMetricIdList)) {
-                  return !blackMetricIdList.includes(item.id);
-                }
-                return false;
-              })}
-              onSubmit={() => {
-                queryThemeListData();
-                onConfigSave?.();
-              }}
-            />
-          </ProCard>
-        </Space>
-      </div>
-    );
-  },
-);
-export default connect(
-  ({ domainManger }: { domainManger: StateType }) => ({
-    domainManger,
-  }),
-  () => {},
-  null,
-  { forwardRef: true },
-)(EntitySection);

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricFieldFormTable.tsx

@@ -67,26 +67,41 @@ const MetricFieldFormTable: React.FC<Props> = ({
     },
   ];
 
+  const handleUpdateKeys = (updateKeys: Record<string, boolean>) => {
+    setSelectedKeysMap(updateKeys);
+    const selectedKeys: string[] = [];
+    const fieldList = Object.entries(updateKeys).reduce((list: any[], item) => {
+      const [fieldName, selected] = item;
+      if (selected) {
+        selectedKeys.push(fieldName);
+        list.push({ fieldName });
+      }
+      return list;
+    }, []);
+    setSelectedKeys(selectedKeys);
+    onFieldChange(fieldList);
+  };
+
   const rowSelection = {
     selectedRowKeys: selectedKeys,
     onSelect: (record: ISemantic.IFieldTypeParamsItem, selected: boolean) => {
       const updateKeys = { ...selectedKeysMap, [record.fieldName]: selected };
-      const selectedKeys: string[] = [];
-      setSelectedKeysMap(updateKeys);
-      const fieldList = Object.entries(updateKeys).reduce((list: any[], item) => {
-        const [fieldName, selected] = item;
-        if (selected) {
-          selectedKeys.push(fieldName);
-          list.push({ fieldName });
-        }
-        return list;
-      }, []);
-      setSelectedKeys(selectedKeys);
-      onFieldChange(fieldList);
+      handleUpdateKeys(updateKeys);
+    },
+    onSelectAll: (
+      selected: boolean,
+      selectedRows: ISemantic.IFieldTypeParamsItem[],
+      changeRows: ISemantic.IFieldTypeParamsItem[],
+    ) => {
+      const updateKeys = changeRows.reduce(
+        (keyMap: Record<string, boolean>, item: ISemantic.IFieldTypeParamsItem) => {
+          keyMap[item.fieldName] = selected;
+          return keyMap;
+        },
+        {},
+      );
+      handleUpdateKeys({ ...selectedKeysMap, ...updateKeys });
     },
-    // onChange: (_selectedRowKeys: any[]) => {
-    //   setSelectedKeys([..._selectedRowKeys]);
-    // },
   };
 
   return (

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMeasuresFormTable.tsx

@@ -106,34 +106,52 @@ const MetricMeasuresFormTable: React.FC<Props> = ({
     },
   ];
 
+  const handleUpdateKeys = (updateKeys: Record<string, boolean>) => {
+    const datasource =
+      datasourceId && Array.isArray(measuresList)
+        ? measuresList.filter((item) => item.datasourceId === datasourceId)
+        : measuresList;
+    setSelectedKeysMap(updateKeys);
+    const selectedKeys: string[] = [];
+    const measures = datasource.reduce(
+      (list: any[], { bizName, name, expr, datasourceId, agg }) => {
+        if (updateKeys[bizName] === true) {
+          selectedKeys.push(bizName);
+          list.push({
+            bizName,
+            name,
+            expr,
+            agg,
+            datasourceId,
+          });
+        }
+        return list;
+      },
+      [],
+    );
+    setSelectedKeys(selectedKeys);
+    onFieldChange(measures);
+  };
+
   const rowSelection = {
     selectedRowKeys: selectedKeys,
     onSelect: (record: ISemantic.IMeasure, selected: boolean) => {
-      const datasource =
-        datasourceId && Array.isArray(measuresList)
-          ? measuresList.filter((item) => item.datasourceId === datasourceId)
-          : measuresList;
       const updateKeys = { ...selectedKeysMap, [record.bizName]: selected };
-      setSelectedKeysMap(updateKeys);
-      const selectedKeys: string[] = [];
-      const measures = datasource.reduce(
-        (list: any[], { bizName, name, expr, datasourceId, agg }) => {
-          if (updateKeys[bizName] === true) {
-            selectedKeys.push(bizName);
-            list.push({
-              bizName,
-              name,
-              expr,
-              agg,
-              datasourceId,
-            });
-          }
-          return list;
+      handleUpdateKeys(updateKeys);
+    },
+    onSelectAll: (
+      selected: boolean,
+      selectedRows: ISemantic.IMeasure[],
+      changeRows: ISemantic.IMeasure[],
+    ) => {
+      const updateKeys = changeRows.reduce(
+        (keyMap: Record<string, boolean>, item: ISemantic.IMeasure) => {
+          keyMap[item.bizName] = selected;
+          return keyMap;
         },
-        [],
+        {},
       );
-      setSelectedKeys(selectedKeys);
-      onFieldChange(measures);
+      handleUpdateKeys({ ...selectedKeysMap, ...updateKeys });
     },
   };
 

webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMetricFormTable.tsx

@@ -74,25 +74,43 @@ const MetricMetricFormTable: React.FC<Props> = ({
     },
   ];
 
+  const handleUpdateKeys = (updateKeys: Record<string, boolean>) => {
+    setSelectedKeysMap(updateKeys);
+    const selectedKeys: string[] = [];
+    const metrics = metricList.reduce((list: any[], item) => {
+      const { bizName, id } = item;
+      if (updateKeys[bizName] === true) {
+        selectedKeys.push(bizName);
+        list.push({
+          bizName,
+          id,
+        });
+      }
+      return list;
+    }, []);
+    setSelectedKeys(selectedKeys);
+    onFieldChange(metrics);
+  };
+
   const rowSelection = {
     selectedRowKeys: selectedKeys,
     onSelect: (record: ISemantic.IMeasure, selected: boolean) => {
       const updateKeys = { ...selectedKeysMap, [record.bizName]: selected };
-      setSelectedKeysMap(updateKeys);
-      const selectedKeys: string[] = [];
-      const metrics = metricList.reduce((list: any[], item) => {
-        const { bizName, id } = item;
-        if (updateKeys[bizName] === true) {
-          selectedKeys.push(bizName);
-          list.push({
-            bizName,
-            id,
-          });
-        }
-        return list;
-      }, []);
-      setSelectedKeys(selectedKeys);
-      onFieldChange(metrics);
+      handleUpdateKeys(updateKeys);
+    },
+    onSelectAll: (
+      selected: boolean,
+      selectedRows: ISemantic.IMetricItem[],
+      changeRows: ISemantic.IMetricItem[],
+    ) => {
+      const updateKeys = changeRows.reduce(
+        (keyMap: Record<string, boolean>, item: ISemantic.IMetricItem) => {
+          keyMap[item.bizName] = selected;
+          return keyMap;
+        },
+        {},
+      );
+      handleUpdateKeys({ ...selectedKeysMap, ...updateKeys });
     },
   };
 

webapp/packages/supersonic-fe/src/pages/SemanticModel/data.d.ts

@@ -57,6 +57,17 @@ export declare namespace IDataSource {
     dataType: string;
     fieldName: string;
   }
+
+  type ISqlParamsValueType = 'STRING' | 'EXPR' | 'NUMBER';
+  interface ISqlParamsItem {
+    index?: number;
+    defaultValues: (boolean | string | number)[];
+    name: string;
+    // type: string;
+    valueType: ISqlParamsValueType;
+    udf?: boolean;
+  }
+
   interface IDataSourceDetail {
     queryType: string;
     sqlQuery: string;
@@ -65,6 +76,7 @@ export declare namespace IDataSource {
     fields: IDataSourceDetailFieldsItem[];
     dimensions: IDimensionsItem[];
     measures: IMeasuresItem[];
+    sqlVariables: ISqlParamsItem[];
   }
 
   interface IDataSourceItem {
@@ -330,6 +342,41 @@ export declare namespace ISemantic {
     description?: string;
   }
   type IDatabaseItemList = IDatabaseItem[];
+
+  interface IDictKnowledgeConfigItemConfig {
+    metricId?: number;
+    blackList: string[];
+    whiteList: string[];
+    ruleList: any[];
+    limit?: number;
+  }
+  interface IDictKnowledgeConfigItem {
+    id: number;
+    modelId: number;
+    bizName: string;
+    type: string;
+    itemId: number;
+    config: IDictKnowledgeConfigItemConfig;
+    status: string;
+    nature?: string;
+  }
+
+  interface IDictKnowledgeTaskItem {
+    id: number;
+    modelId: number;
+    bizName: string;
+    type: string;
+    itemId: number;
+    config: IDictKnowledgeConfigItemConfig;
+    status: string | null;
+    name: string;
+    description: string | null;
+    taskStatus: string;
+    createdAt: string;
+    createdBy: string;
+    elapsedMs: number | null;
+    nature: string;
+  }
 }
 
 export declare namespace IChatConfig {
@@ -345,21 +392,6 @@ export declare namespace IChatConfig {
     };
   }
 
-  interface IConfig {
-    id: any;
-    modelId: number;
-    modelName: string;
-    chatAggRichConfig: IChatRichConfig;
-    chatDetailRichConfig: IChatRichConfig;
-    recommendedQuestions: { question: string }[];
-    bizName: string;
-    statusEnum: string;
-    createdBy: string;
-    updatedBy: string;
-    createdAt: string;
-    updatedAt: string;
-  }
-
   interface IKnowledgeInfosItem {
     itemId: number;
     bizName: string;

webapp/packages/supersonic-fe/src/pages/SemanticModel/enum.ts

@@ -20,11 +20,12 @@ export enum MetricTypeWording {
 }
 
 export enum DictTaskState {
-  ERROR = '错误',
-  PENDING = '等待',
-  RUNNING = '正在执行',
-  SUCCESS = '成功',
-  UNKNOWN = '未知',
+  initial = '--',
+  error = '错误',
+  pending = '等待',
+  running = '正在执行',
+  success = '成功',
+  unknown = '未知',
 }
 
 export enum StatusEnum {
@@ -41,3 +42,22 @@ export enum OperatorEnum {
   IN = 'IN',
   LIKE = 'LIKE',
 }
+
+export enum KnowledgeConfigTypeEnum {
+  DIMENSION = 'DIMENSION',
+  METRIC = 'METRIC',
+  DOMAIN = 'DOMAIN',
+  ENTITY = 'ENTITY',
+  VIEW = 'VIEW',
+  MODEL = 'MODEL',
+  UNKNOWN = 'UNKNOWN',
+}
+
+export enum KnowledgeConfigStatusEnum {
+  ONLINE = 'ONLINE',
+  OFFLINE = 'OFFLINE',
+  DELETED = 'DELETED',
+  INITIALIZED = 'INITIALIZED',
+  UNAVAILABLE = 'UNAVAILABLE',
+  UNKNOWN = 'UNKNOWN',
+}

webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts

@@ -1,6 +1,7 @@
 import request from 'umi-request';
 import moment from 'moment';
 import { DatePeridMap } from '@/pages/SemanticModel/constant';
+import { IDataSource } from './data';
 
 const getRunningEnv = () => {
   return window.location.pathname.includes('/chatSetting/') ? 'chat' : 'semantic';
@@ -256,12 +257,6 @@ export function getDomainExtendConfig(data: any): Promise<any> {
   });
 }
 
-export function getDomainExtendDetailConfig(data: any): Promise<any> {
-  return request(`${process.env.CHAT_API_BASE_URL}conf/richDesc/${data.modelId}`, {
-    method: 'GET',
-  });
-}
-
 export function getDatasourceRelaList(id?: number): Promise<any> {
   return request(`${process.env.API_BASE_URL}datasource/getDatasourceRelaList/${id}`, {
     method: 'GET',
@@ -370,6 +365,7 @@ export function testDatabaseConnect(data: SaveDatabaseParams): Promise<any> {
 type ExcuteSqlParams = {
   sql: string;
   id: number;
+  sqlVariables: IDataSource.ISqlParamsItem[];
 };
 
 // 执行脚本
@@ -444,19 +440,33 @@ export function getMetricsToCreateNewMetric(data: any): Promise<any> {
 }
 
 export function createDictTask(data: any): Promise<any> {
-  return request(`${process.env.CHAT_API_BASE_URL}dict/task`, {
+  return request(`${process.env.API_BASE_URL}knowledge/task`, {
     method: 'POST',
     data,
   });
 }
 
-export function deleteDictTask(data: any): Promise<any> {
-  return request(`${process.env.CHAT_API_BASE_URL}dict/task/delete`, {
+export function createDictConfig(data: any): Promise<any> {
+  return request(`${process.env.API_BASE_URL}knowledge/conf`, {
     method: 'POST',
     data,
   });
 }
 
+export function editDictConfig(data: any): Promise<any> {
+  return request(`${process.env.API_BASE_URL}knowledge/conf`, {
+    method: 'PUT',
+    data,
+  });
+}
+
+export function deleteDictTask(data: any): Promise<any> {
+  return request(`${process.env.API_BASE_URL}knowledge/task/delete`, {
+    method: 'PUT',
+    data,
+  });
+}
+
 export function searchDictLatestTaskList(data: any): Promise<any> {
   return request(`${process.env.API_BASE_URL}knowledge/task/search`, {
     method: 'POST',

webapp/packages/supersonic-fe/src/utils/utils.ts

@@ -451,3 +451,22 @@ export function isArrayOfValues(array: any) {
   }
   return false;
 }
+
+type ObjToArrayParams = Record<string, string>;
+
+const keyTypeTran = {
+  string: String,
+  number: Number,
+};
+/**
+ * obj转成value，label的数组
+ * @param _obj
+ */
+export const objToArray = (_obj: ObjToArrayParams, keyType: string = 'string') => {
+  return Object.keys(_obj).map((key) => {
+    return {
+      value: keyTypeTran[keyType](key),
+      label: _obj[key],
+    };
+  });
+};