(improvement)(launcher) Optimize the demo to prevent the dimension value dictionary from being loaded after the demo is loaded. (#1059)

Co-authored-by: jolunoluo

chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.server.executor;
 
 import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
 import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
+import com.tencent.supersonic.chat.server.util.ResultFormatter;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
@@ -20,7 +21,11 @@ public class SqlExecutor implements ChatExecutor {
         }
         ExecuteQueryReq executeQueryReq = buildExecuteReq(chatExecuteContext);
         ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
-        return chatQueryService.performExecution(executeQueryReq);
+        QueryResult queryResult = chatQueryService.performExecution(executeQueryReq);
+        String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
+                queryResult.getQueryResults());
+        queryResult.setTextResult(textResult);
+        return queryResult;
     }
 
     private ExecuteQueryReq buildExecuteReq(ChatExecuteContext chatExecuteContext) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java

@@ -1,16 +1,22 @@
 package com.tencent.supersonic.chat.server.parser;
 
+import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
 import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
 import com.tencent.supersonic.chat.server.util.QueryReqConverter;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.server.service.ChatQueryService;
-
-import java.util.List;
-
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 @Slf4j
 public class NL2SQLParser implements ChatParser {
@@ -31,6 +37,7 @@ public class NL2SQLParser implements ChatParser {
             parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
         }
         parseResp.getParseTimeCost().setSqlTime(text2SqlParseResp.getParseTimeCost().getSqlTime());
+        formatParseResult(parseResp);
     }
 
     private boolean checkSkip(ParseResp parseResp) {
@@ -43,4 +50,49 @@ public class NL2SQLParser implements ChatParser {
         return false;
     }
 
+    private void formatParseResult(ParseResp parseResp) {
+        List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
+        for (SemanticParseInfo parseInfo : selectedParses) {
+            formatParseInfo(parseInfo);
+        }
+    }
+
+    private void formatParseInfo(SemanticParseInfo parseInfo) {
+        if (!PluginQueryManager.isPluginQuery(parseInfo.getQueryMode())) {
+            formatNL2SQLParseInfo(parseInfo);
+        }
+    }
+
+    private void formatNL2SQLParseInfo(SemanticParseInfo parseInfo) {
+        StringBuilder textBuilder = new StringBuilder();
+        textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
+        Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
+        metric.ifPresent(schemaElement ->
+                textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
+        List<String> dimensionNames = parseInfo.getDimensions().stream()
+                .map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
+        if (!CollectionUtils.isEmpty(dimensionNames)) {
+            textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
+        }
+        textBuilder.append("\n\n**筛选条件:** \n");
+        if (parseInfo.getDateInfo() != null) {
+            textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~")
+                    .append(parseInfo.getDateInfo().getEndDate()).append(" ");
+        }
+        if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
+                || CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
+            Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
+            queryFilters.addAll(parseInfo.getMetricFilters());
+            for (QueryFilter queryFilter : queryFilters) {
+                textBuilder.append("**").append(queryFilter.getName()).append("**")
+                        .append(" ")
+                        .append(queryFilter.getOperator().getValue())
+                        .append(" ")
+                        .append(queryFilter.getValue())
+                        .append(" ");
+            }
+        }
+        parseInfo.setTextInfo(textBuilder.toString());
+    }
+
 }

chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java

@@ -55,8 +55,13 @@ public class WebServiceQuery extends PluginSemanticQuery {
         log.info("webServiceResponse result:{}", JsonUtil.toString(object));
         try {
             Map<String, Object> data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class);
-            queryResult.setQueryResults((List<Map<String, Object>>) data.get("resultList"));
-            queryResult.setQueryColumns((List<QueryColumn>) data.get("columns"));
+            if (data.get("resultList") != null) {
+                queryResult.setQueryResults((List<Map<String, Object>>) data.get("resultList"));
+            }
+            if (data.get("columns") != null) {
+                queryResult.setQueryColumns((List<QueryColumn>) data.get("columns"));
+            }
+            queryResult.setTextResult(String.valueOf(data.get("textInfo")));
             queryResult.setQueryState(QueryState.SUCCESS);
         } catch (Exception e) {
             log.info("webServiceResponse result has an exception:{}", e.getMessage());

chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/PluginRecognizer.java

@@ -86,6 +86,7 @@ public abstract class PluginRecognizer {
         properties.put("name", plugin.getName());
         semanticParseInfo.setProperties(properties);
         semanticParseInfo.setScore(distance);
+        semanticParseInfo.setTextInfo(String.format("将由插件工具**%s**来解答", plugin.getName()));
         fillSemanticParseInfo(semanticParseInfo);
         return semanticParseInfo;
     }

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java

@@ -1,16 +0,0 @@
-package com.tencent.supersonic.chat.server.processor.execute;
-
-import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
-import com.tencent.supersonic.chat.server.util.ResultFormatter;
-import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
-
-public class QueryResultFormatProcessor implements ExecuteResultProcessor {
-
-    @Override
-    public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) {
-        String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
-                queryResult.getQueryResults());
-        queryResult.setTextResult(textResult);
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java

@@ -1,61 +0,0 @@
-package com.tencent.supersonic.chat.server.processor.parse;
-
-import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
-import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
-import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
-import org.springframework.util.CollectionUtils;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class ParseResultFormatProcessor implements ParseResultProcessor {
-
-    @Override
-    public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
-        formatParseResult(parseResp);
-    }
-
-    private void formatParseResult(ParseResp parseResp) {
-        List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
-        for (SemanticParseInfo parseInfo : selectedParses) {
-            formatParseInfo(parseInfo);
-        }
-    }
-
-    private void formatParseInfo(SemanticParseInfo parseInfo) {
-        StringBuilder textBuilder = new StringBuilder();
-        textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
-        Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
-        metric.ifPresent(schemaElement ->
-                textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
-        List<String> dimensionNames = parseInfo.getDimensions().stream()
-                .map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
-        if (!CollectionUtils.isEmpty(dimensionNames)) {
-            textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
-        }
-        textBuilder.append("\n\n**筛选条件:** \n");
-        if (parseInfo.getDateInfo() != null) {
-            textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~")
-                    .append(parseInfo.getDateInfo().getEndDate()).append(" ");
-        }
-        if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
-                || CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
-            Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
-            queryFilters.addAll(parseInfo.getMetricFilters());
-            for (QueryFilter queryFilter : queryFilters) {
-                textBuilder.append("**").append(queryFilter.getName()).append("**")
-                        .append(" ")
-                        .append(queryFilter.getOperator().getValue())
-                        .append(" ")
-                        .append(queryFilter.getValue())
-                        .append(" ");
-            }
-        }
-        parseInfo.setTextInfo(textBuilder.toString());
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.chat.server.util;
 
 import com.tencent.supersonic.common.pojo.QueryColumn;
+import org.springframework.util.CollectionUtils;
 
 import java.util.List;
 import java.util.Map;
@@ -8,6 +9,9 @@ import java.util.Map;
 public class ResultFormatter {
 
     public static String transform2TextNew(List<QueryColumn> queryColumns, List<Map<String, Object>> queryResults) {
+        if (CollectionUtils.isEmpty(queryColumns)) {
+            return "";
+        }
         StringBuilder table = new StringBuilder();
         for (QueryColumn column : queryColumns) {
             String columnName = column.getName();
@@ -18,6 +22,9 @@ public class ResultFormatter {
             table.append("|:---:");
         }
         table.append("|\n");
+        if (queryResults == null) {
+            return table.toString();
+        }
         for (Map<String, Object> row : queryResults) {
             for (QueryColumn column : queryColumns) {
                 String columnKey = column.getNameEn();

headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java

@@ -1,82 +0,0 @@
-package com.tencent.supersonic.headless.server.listener;
-
-
-import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
-import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
-import com.tencent.supersonic.headless.server.service.impl.WordService;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.core.annotation.Order;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-
-@Slf4j
-@Component
-@Order(2)
-public class ApplicationStartedListener implements CommandLineRunner {
-
-    @Autowired
-    private KnowledgeBaseService knowledgeBaseService;
-    @Autowired
-    private WordService wordService;
-
-    @Override
-    public void run(String... args) {
-        updateKnowledgeDimValue();
-    }
-
-    public Boolean updateKnowledgeDimValue() {
-        Boolean isOk = false;
-        try {
-            log.debug("ApplicationStartedInit start");
-
-            List<DictWord> dictWords = wordService.getAllDictWords();
-            wordService.setPreDictWords(dictWords);
-            knowledgeBaseService.reloadAllData(dictWords);
-
-            log.debug("ApplicationStartedInit end");
-            isOk = true;
-        } catch (Exception e) {
-            log.error("ApplicationStartedInit error", e);
-        }
-        return isOk;
-    }
-
-    public Boolean updateKnowledgeDimValueAsync() {
-        CompletableFuture.supplyAsync(() -> {
-            updateKnowledgeDimValue();
-            return null;
-        });
-        return true;
-    }
-
-    /***
-     * reload knowledge task
-     */
-    @Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
-    public void reloadKnowledge() {
-        log.debug("reloadKnowledge start");
-
-        try {
-            List<DictWord> dictWords = wordService.getAllDictWords();
-            List<DictWord> preDictWords = wordService.getPreDictWords();
-
-            if (CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
-                log.debug("dictWords has not changed, reloadKnowledge end");
-                return;
-            }
-            log.info("dictWords has changed");
-            wordService.setPreDictWords(dictWords);
-            knowledgeBaseService.updateOnlineKnowledge(wordService.getAllDictWords());
-        } catch (Exception e) {
-            log.error("reloadKnowledge error", e);
-        }
-
-        log.debug("reloadKnowledge end");
-    }
-}

headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/DictWordLoadStartedListener.java

@@ -0,0 +1,47 @@
+package com.tencent.supersonic.headless.server.listener;
+
+import com.tencent.supersonic.headless.server.service.impl.DictWordService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+@Order(2)
+public class DictWordLoadStartedListener implements CommandLineRunner {
+
+    @Autowired
+    private DictWordService dictWordService;
+
+    @Override
+    public void run(String... args) {
+        updateKnowledgeDimValue();
+    }
+
+    public void updateKnowledgeDimValue() {
+        try {
+            log.debug("ApplicationStartedInit start");
+            dictWordService.loadDictWord();
+            log.debug("ApplicationStartedInit end");
+        } catch (Exception e) {
+            log.error("ApplicationStartedInit error", e);
+        }
+    }
+
+    /***
+     * reload knowledge task
+     */
+    @Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
+    public void reloadKnowledge() {
+        log.debug("reloadKnowledge start");
+        try {
+            dictWordService.reloadDictWord();
+        } catch (Exception e) {
+            log.error("reloadKnowledge error", e);
+        }
+        log.debug("reloadKnowledge end");
+    }
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java

@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
 import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
 import com.tencent.supersonic.headless.core.chat.knowledge.builder.WordBuilderFactory;
 import com.tencent.supersonic.headless.server.service.SchemaService;
 import lombok.extern.slf4j.Slf4j;
@@ -20,13 +21,33 @@ import java.util.stream.Collectors;
 
 @Service
 @Slf4j
-public class WordService {
+public class DictWordService {
 
     @Autowired
     private SchemaService schemaService;
+    @Autowired
+    private KnowledgeBaseService knowledgeBaseService;
 
     private List<DictWord> preDictWords = new ArrayList<>();
 
+    public void loadDictWord() {
+        List<DictWord> dictWords = getAllDictWords();
+        setPreDictWords(dictWords);
+        knowledgeBaseService.reloadAllData(dictWords);
+    }
+
+    public void reloadDictWord() {
+        List<DictWord> dictWords = getAllDictWords();
+        List<DictWord> preDictWords = getPreDictWords();
+        if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
+            log.debug("dictWords has not changed, reloadKnowledge end");
+            return;
+        }
+        log.info("dictWords has changed");
+        setPreDictWords(dictWords);
+        knowledgeBaseService.updateOnlineKnowledge(getAllDictWords());
+    }
+
     public List<DictWord> getAllDictWords() {
         SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema());
 

launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.service.TagObjectService;
 import com.tencent.supersonic.headless.server.service.TermService;
+import com.tencent.supersonic.headless.server.service.impl.DictWordService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -86,6 +87,8 @@ public abstract class S2BaseDemo implements CommandLineRunner {
     protected SysParameterService sysParameterService;
     @Autowired
     protected CanvasService canvasService;
+    @Autowired
+    protected DictWordService dictWordService;
     @Value("${s2.demo.names:S2VisitsDemo}")
     protected List<String> demoList;
     @Value("${s2.demo.enableLLM:true}")

launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java

@@ -13,7 +13,6 @@ import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
 import com.tencent.supersonic.chat.server.agent.RuleParserTool;
 import com.tencent.supersonic.chat.server.plugin.Plugin;
 import com.tencent.supersonic.chat.server.plugin.PluginParseConfig;
-import com.tencent.supersonic.chat.server.plugin.build.ParamOption;
 import com.tencent.supersonic.chat.server.plugin.build.WebBase;
 import com.tencent.supersonic.common.pojo.JoinCondition;
 import com.tencent.supersonic.common.pojo.ModelRela;
@@ -111,9 +110,12 @@ public class S2VisitsDemo extends S2BaseDemo {
             //create terms and plugin
             addTerm(s2Domain);
             addTerm_1(s2Domain);
-            addPlugin(s2DataSet, userDimension, userModel);
+            addPlugin(s2DataSet);
             addSysParameter();
 
+            //load dict word
+            loadDictWord();
+
             //create agent
             Integer agentId = addAgent(s2DataSet.getId());
             addSampleChats(agentId);
@@ -531,8 +533,7 @@ public class S2VisitsDemo extends S2BaseDemo {
         authService.addOrUpdateAuthGroup(authGroupReq);
     }
 
-    private void addPlugin(DataSetResp s2DataSet, DimensionResp userDimension,
-                           ModelResp userModel) {
+    private void addPlugin(DataSetResp s2DataSet) {
         Plugin plugin1 = new Plugin();
         plugin1.setType("WEB_PAGE");
         plugin1.setDataSetList(Arrays.asList(s2DataSet.getId()));
@@ -545,13 +546,7 @@ public class S2VisitsDemo extends S2BaseDemo {
         plugin1.setParseModeConfig(JSONObject.toJSONString(pluginParseConfig));
         WebBase webBase = new WebBase();
         webBase.setUrl("www.yourbi.com");
-        ParamOption paramOption = new ParamOption();
-        paramOption.setKey("name");
-        paramOption.setParamType(ParamOption.ParamType.SEMANTIC);
-        paramOption.setElementId(userDimension.getId());
-        paramOption.setModelId(userModel.getId());
-        List<ParamOption> paramOptions = Arrays.asList(paramOption);
-        webBase.setParamOptions(paramOptions);
+        webBase.setParamOptions(Lists.newArrayList());
         plugin1.setConfig(JsonUtil.toString(webBase));
         pluginService.createPlugin(plugin1, user);
     }
@@ -565,4 +560,8 @@ public class S2VisitsDemo extends S2BaseDemo {
         return tagObjectService.create(tagObjectReq, user);
     }
 
+    private void loadDictWord() {
+        dictWordService.loadDictWord();
+    }
+
 }

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -60,14 +60,12 @@ com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\
 com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
     com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\
     com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\
-    com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor,\
-    com.tencent.supersonic.chat.server.processor.parse.ParseResultFormatProcessor
+    com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor
 
 com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
     com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
     com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
-    com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor,\
-    com.tencent.supersonic.chat.server.processor.execute.QueryResultFormatProcessor
+    com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
 
 com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
     com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore

launchers/standalone/src/main/resources/application-local.yaml

@@ -61,7 +61,7 @@ s2:
   embedding:
     url: ${s2.pyllm.url}
     persistent:
-      path: /tmp
+      path: d://
 
   demo:
     names: S2VisitsDemo,S2ArtistDemo