Refactor translator module (#1932)

* [improvement][Chat] Support agent permission management #1143 * [improvement][chat]Iterate LLM prompts of parsing and correction. * [improvement][headless]Clean code logic of headless core. * (fix) (chat) 记忆管理更新不生效 (#1912) * [improvement][headless-fe] Added null-check conditions to the data formatting function. * [improvement][headless]Clean code logic of headless translator. * [improvement][headless-fe] Added permissions management for agents. * [improvement][headless-fe] Unified the assistant's permission settings interaction to match the system style. * [improvement](Dict)Support returns dict task list of dimensions by page * [improvement][headless-fe] Revised the interaction for semantic modeling routing and implemented the initial version of metric management switching. * [improvement][launcher]Set system property `s2.test` in junit tests in order to facilitate conditional breakpoints. * [improvement][headless] add validateAndQuery interface in SqlQueryApiController * [improvement][launcher]Use API to get element ID avoiding hard-code. * [improvement][launcher]Support DuckDB database and refactor translator code structure. --------- Co-authored-by: lxwcodemonkey <jolunoluo@tencent.com> Co-authored-by: tristanliu <tristanliu@tencent.com> Co-authored-by: daikon12 <1059907724@qq.com> Co-authored-by: lexluo09 <39718951+lexluo09@users.noreply.github.com>
2026-04-26 18:24:20 +08:00 · 2024-11-30 00:27:33 +08:00
parent 224c114d20
commit 593597fe26
155 changed files with 3757 additions and 3551 deletions
--- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java
@@ -108,9 +108,9 @@ public abstract class S2BaseDemo implements CommandLineRunner {
        }
    }

-    abstract void doRun();
+    protected abstract void doRun();

-    abstract boolean checkNeedToRun();
+    protected abstract boolean checkNeedToRun();

    protected DatabaseResp addDatabaseIfNotExist() {
        List<DatabaseResp> databaseList = databaseService.getDatabaseList(defaultUser);
@@ -119,8 +119,8 @@ public abstract class S2BaseDemo implements CommandLineRunner {
        }
        String url = dataSourceProperties.getUrl();
        DatabaseReq databaseReq = new DatabaseReq();
-        databaseReq.setName("H2数据库DEMO");
-        databaseReq.setDescription("样例数据库实例仅用于体验，正式使用请切换持久化数据库");
+        databaseReq.setName("S2数据库DEMO");
+        databaseReq.setDescription("样例数据库实例仅用于体验");
        if (StringUtils.isNotBlank(url)
                && url.toLowerCase().contains(DataType.MYSQL.getFeature().toLowerCase())) {
            databaseReq.setType(DataType.MYSQL.getFeature());
--- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java
@@ -7,26 +7,12 @@ import com.tencent.supersonic.chat.server.agent.Agent;
 import com.tencent.supersonic.chat.server.agent.AgentToolType;
 import com.tencent.supersonic.chat.server.agent.DatasetTool;
 import com.tencent.supersonic.chat.server.agent.ToolConfig;
-import com.tencent.supersonic.chat.server.processor.execute.DataInterpretProcessor;
 import com.tencent.supersonic.common.pojo.ChatApp;
 import com.tencent.supersonic.common.pojo.JoinCondition;
 import com.tencent.supersonic.common.pojo.ModelRela;
-import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.AppModule;
-import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.TimeMode;
-import com.tencent.supersonic.common.pojo.enums.TypeEnums;
+import com.tencent.supersonic.common.pojo.enums.*;
 import com.tencent.supersonic.common.util.ChatAppManager;
-import com.tencent.supersonic.headless.api.pojo.AggregateTypeDefaultConfig;
-import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
-import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
-import com.tencent.supersonic.headless.api.pojo.Dim;
-import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
-import com.tencent.supersonic.headless.api.pojo.Identify;
-import com.tencent.supersonic.headless.api.pojo.Measure;
-import com.tencent.supersonic.headless.api.pojo.ModelDetail;
-import com.tencent.supersonic.headless.api.pojo.QueryConfig;
-import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
+import com.tencent.supersonic.headless.api.pojo.*;
 import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
 import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
@@ -40,11 +26,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;

-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;

@Component
@Slf4j
@@ -59,8 +41,8 @@ public class S2CompanyDemo extends S2BaseDemo {
            ModelResp model_brand = addModel_2(domain, demoDatabase);
            ModelResp model_brand_revenue = addModel_3(domain, demoDatabase);

-            addModelRela(domain, model_company, model_brand, "company_id");
-            addModelRela(domain, model_brand, model_brand_revenue, "brand_id");
+            addModelRela(domain, model_brand, model_company, "company_id");
+            addModelRela(domain, model_brand_revenue, model_brand, "brand_id");

            DataSetResp dataset = addDataSet(domain);
            addAgent(dataset.getId());
@@ -70,7 +52,7 @@ public class S2CompanyDemo extends S2BaseDemo {
    }

    @Override
-    boolean checkNeedToRun() {
+    protected boolean checkNeedToRun() {
        List<DomainResp> domainList = domainService.getDomainList();
        for (DomainResp domainResp : domainList) {
            if (domainResp.getBizName().equalsIgnoreCase("corporate")) {
@@ -124,8 +106,7 @@ public class S2CompanyDemo extends S2BaseDemo {
        modelDetail.setMeasures(measures);

        modelDetail.setQueryType("sql_query");
-        modelDetail.setSqlQuery("SELECT company_id,company_name,headquarter_address,"
-                + "company_established_time,founder,ceo,annual_turnover,employee_count FROM company");
+        modelDetail.setSqlQuery("SELECT * FROM company");
        modelReq.setModelDetail(modelDetail);
        ModelResp companyModel = modelService.createModel(modelReq, defaultUser);

@@ -164,8 +145,7 @@ public class S2CompanyDemo extends S2BaseDemo {
        modelDetail.setMeasures(measures);

        modelDetail.setQueryType("sql_query");
-        modelDetail.setSqlQuery("SELECT brand_id,brand_name,brand_established_time,"
-                + "company_id,legal_representative,registered_capital FROM brand");
+        modelDetail.setSqlQuery("SELECT * FROM brand");
        modelReq.setModelDetail(modelDetail);
        ModelResp brandModel = modelService.createModel(modelReq, defaultUser);

@@ -205,8 +185,7 @@ public class S2CompanyDemo extends S2BaseDemo {
        modelDetail.setMeasures(measures);

        modelDetail.setQueryType("sql_query");
-        modelDetail.setSqlQuery("SELECT year_time,brand_id,revenue,profit,"
-                + "revenue_growth_year_on_year,profit_growth_year_on_year FROM brand_revenue");
+        modelDetail.setSqlQuery("SELECT * FROM brand_revenue");
        modelReq.setModelDetail(modelDetail);
        return modelService.createModel(modelReq, defaultUser);
    }
@@ -245,7 +224,7 @@ public class S2CompanyDemo extends S2BaseDemo {
        modelRelaReq.setDomainId(domain.getId());
        modelRelaReq.setFromModelId(fromModel.getId());
        modelRelaReq.setToModelId(toModel.getId());
-        modelRelaReq.setJoinType("left join");
+        modelRelaReq.setJoinType("inner join");
        modelRelaReq.setJoinConditions(joinConditions);
        modelRelaService.save(modelRelaReq, defaultUser);
    }
@@ -272,7 +251,6 @@ public class S2CompanyDemo extends S2BaseDemo {
        Map<String, ChatApp> chatAppConfig =
                Maps.newHashMap(ChatAppManager.getAllApps(AppModule.CHAT));
        chatAppConfig.values().forEach(app -> app.setChatModelId(demoChatModel.getId()));
-        chatAppConfig.get(DataInterpretProcessor.APP_KEY).setEnable(true);
        agent.setChatAppConfig(chatAppConfig);

        agentService.createAgent(agent, defaultUser);
--- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SingerDemo.java
@@ -57,7 +57,7 @@ public class S2SingerDemo extends S2BaseDemo {
    }

    @Override
-    boolean checkNeedToRun() {
+    protected boolean checkNeedToRun() {
        List<DomainResp> domainList = domainService.getDomainList();
        for (DomainResp domainResp : domainList) {
            if (domainResp.getBizName().equalsIgnoreCase("singer")) {
--- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2SmallTalkDemo.java
@@ -46,9 +46,10 @@ public class S2SmallTalkDemo extends S2BaseDemo {
    }

    @Override
-    boolean checkNeedToRun() {
+    protected boolean checkNeedToRun() {
        List<String> agentNames =
                agentService.getAgents().stream().map(Agent::getName).collect(Collectors.toList());
        return !agentNames.contains("闲聊助手");
    }
+
 }
--- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
@@ -79,8 +79,8 @@ public class S2VisitsDemo extends S2BaseDemo {
            ModelResp userModel = addModel_1(s2Domain, demoDatabase);
            ModelResp pvUvModel = addModel_2(s2Domain, demoDatabase);
            ModelResp stayTimeModel = addModel_3(s2Domain, demoDatabase);
-            addModelRela(s2Domain, userModel, pvUvModel, "user_name");
-            addModelRela(s2Domain, userModel, stayTimeModel, "user_name");
+            addModelRela(s2Domain, pvUvModel, userModel, "user_name");
+            addModelRela(s2Domain, stayTimeModel, userModel, "user_name");

            // create metrics and dimensions
            DimensionResp departmentDimension = getDimension("department", userModel);
@@ -146,7 +146,8 @@ public class S2VisitsDemo extends S2BaseDemo {
        agent.setStatus(1);
        agent.setEnableSearch(1);
        agent.setExamples(Lists.newArrayList("近15天超音数访问次数汇总", "按部门统计超音数的访问人数", "对比alice和lucy的停留时长",
-                "过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长"));
+                "过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长",
+                "今年以来访问次数最高的一天是哪一天"));

        // configure tools
        ToolConfig toolConfig = new ToolConfig();
@@ -198,6 +199,7 @@ public class S2VisitsDemo extends S2BaseDemo {

        List<Dim> dimensions = new ArrayList<>();
        dimensions.add(new Dim("部门", "department", DimensionType.categorical, 1));
+        // dimensions.add(new Dim("用户", "user_name", DimensionType.categorical, 1));
        modelDetail.setDimensions(dimensions);
        List<Field> fields = Lists.newArrayList();
        fields.add(Field.builder().fieldName("user_name").dataType("Varchar").build());
@@ -382,9 +384,9 @@ public class S2VisitsDemo extends S2BaseDemo {
        metricReq.setDescription("访问的用户个数");
        metricReq.setAlias("UV,访问人数");
        MetricDefineByFieldParams metricTypeParams = new MetricDefineByFieldParams();
-        metricTypeParams.setExpr("count(distinct user_id)");
+        metricTypeParams.setExpr("count(distinct user_name)");
        List<FieldParam> fieldParams = new ArrayList<>();
-        fieldParams.add(new FieldParam("user_id"));
+        fieldParams.add(new FieldParam("user_name"));
        metricTypeParams.setFields(fieldParams);
        metricReq.setMetricDefineByFieldParams(metricTypeParams);
        metricReq.setMetricDefineType(MetricDefineType.FIELD);
--- a/launchers/standalone/src/main/resources/META-INF/spring.factories
+++ b/launchers/standalone/src/main/resources/META-INF/spring.factories
@@ -26,15 +26,18 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\

 com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
    com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
-    com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\
-    com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\
-    com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter
+    com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\
+    com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter,\
+    com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
+    com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter

-com.tencent.supersonic.headless.core.translator.QueryOptimizer=\
-    com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer
+com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\
+    com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\
+    com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\
+    com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer

-com.tencent.supersonic.headless.core.translator.QueryParser=\
-    com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser
+com.tencent.supersonic.headless.core.translator.parser.QueryParser=\
+    com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser

 com.tencent.supersonic.headless.core.executor.QueryExecutor=\
    com.tencent.supersonic.headless.core.executor.JdbcExecutor
--- a/launchers/standalone/src/main/resources/application-local.yaml
+++ b/launchers/standalone/src/main/resources/application-local.yaml
@@ -11,4 +11,24 @@ spring:
  h2:
    console:
      path: /h2-console/semantic
-      enabled: true
+      enabled: true
+
+### Comment out following lines if using MySQL
+#spring:
+#  datasource:
+#    driver-class-name: com.mysql.cj.jdbc.Driver
+#    url: jdbc:mysql://localhost:3306/s2_database?user=root
+#    username: root
+#    password:
+#  sql:
+#    enabled: true
+#    mode: always
+#    username: root
+#    password:
+#    init:
+#      schema-locations: classpath:db/schema-mysql.sql,classpath:db/schema-mysql-demo.sql
+#      data-locations: classpath:db/data-mysql.sql,classpath:db/data-mysql-demo.sql
+#  h2:
+#    console:
+#      path: /h2-console/semantic
+#      enabled: true
--- a/launchers/standalone/src/main/resources/config.update/sql-update.sql
+++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql
@@ -396,5 +396,5 @@ ALTER TABLE s2_agent DROP COLUMN  `enable_memory_review`;
 alter table s2_agent add column `enable_feedback` tinyint DEFAULT 1;

 --20241116
-alter table s2_agent add column `admin` varchar(1000);
-alter table s2_agent add column `viewer` varchar(1000);
+alter table s2_agent add column `admin` varchar(1000) COLLATE utf8_unicode_ci DEFAULT NULL;
+alter table s2_agent add column `viewer` varchar(1000) COLLATE utf8_unicode_ci DEFAULT NULL;
--- a/launchers/standalone/src/main/resources/db/data-mysql-demo.sql
+++ b/launchers/standalone/src/main/resources/db/data-mysql-demo.sql
@@ -1,4 +1,4 @@
-------S2VisitsDemo
+-- S2VisitsDemo
 insert into s2_user_department (user_name, department) values ('jack','HR');
 insert into s2_user_department (user_name, department) values ('tom','sales');
 insert into s2_user_department (user_name, department) values ('lucy','marketing');
@@ -1019,7 +1019,7 @@ INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (
 INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (DATE_SUB(CURRENT_DATE(), INTERVAL 15 DAY), 'lucy', '0.8124302447925607', 'p4');
 INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (DATE_SUB(CURRENT_DATE(), INTERVAL 8 DAY), 'lucy', '0.039935860913407284', 'p2');

-------S2ArtistDemo
+-- S2ArtistDemo
 INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('周杰伦', '港台', '青花瓷', '国风', 1000000, 1000000, 1000000);

--- a/launchers/standalone/src/main/resources/db/schema-mysql-demo.sql
+++ b/launchers/standalone/src/main/resources/db/schema-mysql-demo.sql
@@ -1,4 +1,4 @@
-------S2VisitsDemo
+-- S2VisitsDemo
 CREATE TABLE IF NOT EXISTS `s2_user_department` (
      `user_name` varchar(200) NOT NULL,
       `department` varchar(200) NOT NULL
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS `singer` (
    `favor_cnt` bigint DEFAULT NULL
 )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

-------S2ArtistDemo
+-- S2ArtistDemo
 CREATE TABLE IF NOT EXISTS `genre` (
    `g_name` varchar(20) NOT NULL , -- genre name
    `rating` INT ,
--- a/launchers/standalone/src/main/resources/db/schema-mysql.sql
+++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql
@@ -15,6 +15,8 @@ CREATE TABLE IF NOT EXISTS `s2_agent` (
    `created_at` datetime DEFAULT NULL,
    `updated_by` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
    `updated_at` datetime DEFAULT NULL,
+    `admin` varchar(1000) COLLATE utf8_unicode_ci DEFAULT NULL,
+    `viewer` varchar(1000) COLLATE utf8_unicode_ci DEFAULT NULL,
    PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

@@ -539,7 +541,7 @@ CREATE TABLE IF NOT EXISTS `s2_term` (
    PRIMARY KEY (`id`)
 ) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='术语表';

-CREATE TABLE `s2_user_token` (
+CREATE TABLE IF NOT EXISTS `s2_user_token` (
     `id` bigint NOT NULL AUTO_INCREMENT,
     `name` VARCHAR(255) NOT NULL,
     `user_name` VARCHAR(255)  NOT NULL,