(improvement)(chat) Merge ParseResponder into PostProcessor, and change ExecuteResponder to QueryResponder (#448)

Co-authored-by: jolunoluo

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/EntityInfoPostProcessor.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.parse;
+package com.tencent.supersonic.chat.postprocessor;
 
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -12,13 +12,14 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
 import com.tencent.supersonic.chat.service.SemanticService;
 import com.tencent.supersonic.common.util.ContextUtils;
 import org.springframework.util.CollectionUtils;
+
 import java.util.List;
 import java.util.stream.Collectors;
 
-public class EntityInfoParseResponder implements ParseResponder {
+public class EntityInfoPostProcessor implements PostProcessor {
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
         if (CollectionUtils.isEmpty(semanticQueries)) {
             return;

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java

@@ -3,12 +3,14 @@ package com.tencent.supersonic.chat.postprocessor;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
+import com.tencent.supersonic.chat.api.pojo.ChatContext;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
 import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
+import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import com.tencent.supersonic.chat.service.SemanticService;
 import com.tencent.supersonic.common.pojo.QueryType;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -31,7 +33,7 @@ import java.util.stream.Collectors;
 public class MetricCheckPostProcessor implements PostProcessor {
 
     @Override
-    public void process(QueryContext queryContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
         SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
         SemanticSchema semanticSchema = semanticService.getSemanticSchema();

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java

@@ -2,11 +2,13 @@ package com.tencent.supersonic.chat.postprocessor;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
+import com.tencent.supersonic.chat.api.pojo.ChatContext;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
+import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.QueryType;
@@ -16,6 +18,11 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.knowledge.service.SchemaService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -24,10 +31,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.util.CollectionUtils;
 
 /**
  * update parse info from correct sql
@@ -36,7 +39,7 @@ import org.springframework.util.CollectionUtils;
 public class ParseInfoUpdateProcessor implements PostProcessor {
 
     @Override
-    public void process(QueryContext queryContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
         if (CollectionUtils.isEmpty(candidateQueries)) {
             return;

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/PostProcessor.java

@@ -1,5 +1,7 @@
 package com.tencent.supersonic.chat.postprocessor;
+import com.tencent.supersonic.chat.api.pojo.ChatContext;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
+import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 
 /**
  * A post processor do some logic after parser and corrector
@@ -7,6 +9,6 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
 
 public interface PostProcessor {
 
-    void process(QueryContext queryContext);
+    void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);
 
 }

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/QueryRankPostProcessor.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.parse;
+package com.tencent.supersonic.chat.postprocessor;
 
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -18,12 +18,12 @@ import java.util.stream.Collectors;
  * Rank queries by score.
  */
 @Slf4j
-public class QueryRankParseResponder implements ParseResponder {
+public class QueryRankPostProcessor implements PostProcessor {
 
     private static final int candidateTopSize = 5;
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
         candidateQueries = rank(candidateQueries);
         queryContext.setCandidateQueries(candidateQueries);

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/RespBuildPostProcessor.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.parse;
+package com.tencent.supersonic.chat.postprocessor;
 
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,10 +13,10 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 @Slf4j
-public class RespBuildParseResponder implements ParseResponder {
+public class RespBuildPostProcessor implements PostProcessor {
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         QueryReq queryReq = queryContext.getRequest();
         parseResp.setChatId(queryReq.getChatId());
         parseResp.setQueryText(queryReq.getQueryText());

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/SqlOutputPostProcessor.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.parse;
+package com.tencent.supersonic.chat.postprocessor;
 
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,10 +13,10 @@ import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-public class SqlOutputParseResponder implements ParseResponder {
+public class SqlOutputPostProcessor implements PostProcessor {
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         QueryReq queryReq = queryContext.getRequest();
         List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
         if (CollectionUtils.isEmpty(semanticQueries)) {

chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/TimeCostPostProcessor.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.parse;
+package com.tencent.supersonic.chat.postprocessor;
 
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
@@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class TimeCostParseResponder implements ParseResponder {
+public class TimeCostPostProcessor implements PostProcessor {
 
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
+    public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
         long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
         parseResp.getParseTimeCost().setParseTime(
                 System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime());

chat/core/src/main/java/com/tencent/supersonic/chat/responder/EntityInfoQueryResponder.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.execute;
+package com.tencent.supersonic.chat.responder;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
 import com.tencent.supersonic.chat.service.SemanticService;
 import com.tencent.supersonic.common.util.ContextUtils;
 
-public class EntityInfoExecuteResponder implements ExecuteResponder {
+public class EntityInfoQueryResponder implements QueryResponder {
 
     @Override
     public void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {

chat/core/src/main/java/com/tencent/supersonic/chat/responder/QueryResponder.java

@@ -1,10 +1,10 @@
-package com.tencent.supersonic.chat.responder.execute;
+package com.tencent.supersonic.chat.responder;
 
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 
-public interface ExecuteResponder {
+public interface QueryResponder {
 
     void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq);
 

chat/core/src/main/java/com/tencent/supersonic/chat/responder/SimilarMetricQueryResponder.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.responder.execute;
+package com.tencent.supersonic.chat.responder;
 
 import com.alibaba.fastjson.JSONObject;
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
@@ -22,7 +22,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public class SimilarMetricExecuteResponder implements ExecuteResponder {
+public class SimilarMetricQueryResponder implements QueryResponder {
 
     private static final int METRIC_RECOMMEND_SIZE = 5;
 

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ParseResponder.java

@@ -1,11 +0,0 @@
-package com.tencent.supersonic.chat.responder.parse;
-
-import com.tencent.supersonic.chat.api.pojo.ChatContext;
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
-
-public interface ParseResponder {
-
-    void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);
-
-}

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -32,8 +32,7 @@ import com.tencent.supersonic.chat.postprocessor.PostProcessor;
 import com.tencent.supersonic.chat.query.QueryManager;
 import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
 import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
-import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
+import com.tencent.supersonic.chat.responder.QueryResponder;
-import com.tencent.supersonic.chat.responder.parse.ParseResponder;
 import com.tencent.supersonic.chat.service.ChatService;
 import com.tencent.supersonic.chat.service.QueryService;
 import com.tencent.supersonic.chat.service.SemanticService;
@@ -113,8 +112,7 @@ public class QueryServiceImpl implements QueryService {
     private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
     private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
     private List<PostProcessor> postProcessors = ComponentFactory.getPostProcessors();
-    private List<ParseResponder> parseResponders = ComponentFactory.getParseResponders();
+    private List<QueryResponder> executeResponders = ComponentFactory.getExecuteResponders();
-    private List<ExecuteResponder> executeResponders = ComponentFactory.getExecuteResponders();
     private List<SemanticCorrector> semanticCorrectors = ComponentFactory.getSqlCorrections();
 
     @Override
@@ -159,21 +157,12 @@ public class QueryServiceImpl implements QueryService {
         //5. postProcessor
         postProcessors.forEach(postProcessor -> {
             long startTime = System.currentTimeMillis();
-            postProcessor.process(queryCtx);
+            postProcessor.process(parseResult, queryCtx, chatCtx);
             timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
                     .interfaceName(postProcessor.getClass().getSimpleName())
                     .type(CostType.POSTPROCESSOR.getType()).build());
             log.info("{} result:{}", postProcessor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
         });
-        //6. responder
-        parseResponders.forEach(parseResponder -> {
-            long startTime = System.currentTimeMillis();
-            parseResponder.fillResponse(parseResult, queryCtx, chatCtx);
-            timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
-                    .interfaceName(parseResponder.getClass().getSimpleName())
-                    .type(CostType.PARSERRESPONDER.getType()).build());
-            log.info("{} result:{}", parseResponder.getClass().getSimpleName(), JsonUtil.toString(parseResult));
-        });
 
         if (Objects.nonNull(parseResult.getQueryId()) && timeCostDOList.size() > 0) {
             saveTimeCostInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(),
@@ -217,7 +206,7 @@ public class QueryServiceImpl implements QueryService {
             }
             chatCtx.setQueryText(queryReq.getQueryText());
             chatCtx.setUser(queryReq.getUser().getName());
-            for (ExecuteResponder executeResponder : executeResponders) {
+            for (QueryResponder executeResponder : executeResponders) {
                 executeResponder.fillResponse(queryResult, parseInfo, queryReq);
             }
             chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);

chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java

@@ -7,8 +7,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser;
 import com.tencent.supersonic.chat.parser.LLMInterpreter;
 import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
 import com.tencent.supersonic.chat.postprocessor.PostProcessor;
-import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
+import com.tencent.supersonic.chat.responder.QueryResponder;
-import com.tencent.supersonic.chat.responder.parse.ParseResponder;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.core.io.support.SpringFactoriesLoader;
 import java.util.ArrayList;
@@ -24,8 +23,7 @@ public class ComponentFactory {
 
     private static LLMInterpreter llmInterpreter;
     private static List<PostProcessor> postProcessors = new ArrayList<>();
-    private static List<ParseResponder> parseResponders = new ArrayList<>();
+    private static List<QueryResponder> executeResponders = new ArrayList<>();
-    private static List<ExecuteResponder> executeResponders = new ArrayList<>();
     private static ModelResolver modelResolver;
 
     public static List<SchemaMapper> getSchemaMappers() {
@@ -45,13 +43,9 @@ public class ComponentFactory {
         return CollectionUtils.isEmpty(postProcessors) ? init(PostProcessor.class, postProcessors) : postProcessors;
     }
 
-    public static List<ParseResponder> getParseResponders() {
+    public static List<QueryResponder> getExecuteResponders() {
-        return CollectionUtils.isEmpty(parseResponders) ? init(ParseResponder.class, parseResponders) : parseResponders;
-    }
-
-    public static List<ExecuteResponder> getExecuteResponders() {
         return CollectionUtils.isEmpty(executeResponders)
-                ? init(ExecuteResponder.class, executeResponders) : executeResponders;
+                ? init(QueryResponder.class, executeResponders) : executeResponders;
     }
 
     public static SemanticInterpreter getSemanticLayer() {

launchers/chat/src/main/resources/META-INF/spring.factories

@@ -34,9 +34,15 @@ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor
 com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
     com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
 
-com.tencent.supersonic.chat.responder.parse.ParseResponder=\
+com.tencent.supersonic.chat.postprocessor.PostProcessor=\
-    com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
+    com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
-    com.tencent.supersonic.chat.responder.parse.SqlInfoParseResponder
+    com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
+    com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
 
-com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
+com.tencent.supersonic.chat.responder.QueryResponder=\
-    com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder
+    com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
+    com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -38,15 +38,13 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
 
 com.tencent.supersonic.chat.postprocessor.PostProcessor=\
     com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
-    com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor
+    com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
+    com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
 
-com.tencent.supersonic.chat.responder.parse.ParseResponder=\
+com.tencent.supersonic.chat.responder.QueryResponder=\
-    com.tencent.supersonic.chat.responder.parse.QueryRankParseResponder, \
+    com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
-    com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
+    com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder
-    com.tencent.supersonic.chat.responder.parse.SqlOutputParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.TimeCostParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.RespBuildParseResponder
-
-com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
-    com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder, \
-    com.tencent.supersonic.chat.responder.execute.SimilarMetricExecuteResponder

launchers/standalone/src/test/resources/META-INF/spring.factories

@@ -34,15 +34,14 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
 
 com.tencent.supersonic.chat.postprocessor.PostProcessor=\
     com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
-    com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor
+    com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
+    com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
+    com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
 
-com.tencent.supersonic.chat.responder.parse.ParseResponder=\
-    com.tencent.supersonic.chat.responder.parse.QueryRankParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.SqlOutputParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.TimeCostParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.RespBuildParseResponder
 
-com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
+com.tencent.supersonic.chat.responder.QueryResponder=\
-    com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder, \
+    com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
-    com.tencent.supersonic.chat.responder.execute.SimilarMetricExecuteResponder
+    com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder