(improvement)(chat) Merge ParseResponder into PostProcessor, and change ExecuteResponder to QueryResponder (#448)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-29 10:08:25 +08:00
committed by GitHub
parent 3a30a1a317
commit 95e3138ab2
17 changed files with 73 additions and 90 deletions

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.parse;
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -12,13 +12,14 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
public class EntityInfoParseResponder implements ParseResponder {
public class EntityInfoPostProcessor implements PostProcessor {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
if (CollectionUtils.isEmpty(semanticQueries)) {
return;

View File

@@ -3,12 +3,14 @@ package com.tencent.supersonic.chat.postprocessor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -31,7 +33,7 @@ import java.util.stream.Collectors;
public class MetricCheckPostProcessor implements PostProcessor {
@Override
public void process(QueryContext queryContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();

View File

@@ -2,11 +2,13 @@ package com.tencent.supersonic.chat.postprocessor;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
@@ -16,6 +18,11 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.knowledge.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -24,10 +31,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
/**
* update parse info from correct sql
@@ -36,7 +39,7 @@ import org.springframework.util.CollectionUtils;
public class ParseInfoUpdateProcessor implements PostProcessor {
@Override
public void process(QueryContext queryContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
if (CollectionUtils.isEmpty(candidateQueries)) {
return;

View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
/**
* A post processor do some logic after parser and corrector
@@ -7,6 +9,6 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
public interface PostProcessor {
void process(QueryContext queryContext);
void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.parse;
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -18,12 +18,12 @@ import java.util.stream.Collectors;
* Rank queries by score.
*/
@Slf4j
public class QueryRankParseResponder implements ParseResponder {
public class QueryRankPostProcessor implements PostProcessor {
private static final int candidateTopSize = 5;
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
candidateQueries = rank(candidateQueries);
queryContext.setCandidateQueries(candidateQueries);

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.parse;
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,10 +13,10 @@ import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class RespBuildParseResponder implements ParseResponder {
public class RespBuildPostProcessor implements PostProcessor {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
QueryReq queryReq = queryContext.getRequest();
parseResp.setChatId(queryReq.getChatId());
parseResp.setQueryText(queryReq.getQueryText());

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.parse;
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,10 +13,10 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class SqlOutputParseResponder implements ParseResponder {
public class SqlOutputPostProcessor implements PostProcessor {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
QueryReq queryReq = queryContext.getRequest();
List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
if (CollectionUtils.isEmpty(semanticQueries)) {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.parse;
package com.tencent.supersonic.chat.postprocessor;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
@@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TimeCostParseResponder implements ParseResponder {
public class TimeCostPostProcessor implements PostProcessor {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
parseResp.getParseTimeCost().setParseTime(
System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime());

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.execute;
package com.tencent.supersonic.chat.responder;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
public class EntityInfoExecuteResponder implements ExecuteResponder {
public class EntityInfoQueryResponder implements QueryResponder {
@Override
public void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {

View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.responder.execute;
package com.tencent.supersonic.chat.responder;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
public interface ExecuteResponder {
public interface QueryResponder {
void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq);

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.responder.execute;
package com.tencent.supersonic.chat.responder;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
@@ -22,7 +22,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class SimilarMetricExecuteResponder implements ExecuteResponder {
public class SimilarMetricQueryResponder implements QueryResponder {
private static final int METRIC_RECOMMEND_SIZE = 5;

View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.chat.responder.parse;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
public interface ParseResponder {
void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);
}

View File

@@ -32,8 +32,7 @@ import com.tencent.supersonic.chat.postprocessor.PostProcessor;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.responder.QueryResponder;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.chat.service.SemanticService;
@@ -113,8 +112,7 @@ public class QueryServiceImpl implements QueryService {
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
private List<PostProcessor> postProcessors = ComponentFactory.getPostProcessors();
private List<ParseResponder> parseResponders = ComponentFactory.getParseResponders();
private List<ExecuteResponder> executeResponders = ComponentFactory.getExecuteResponders();
private List<QueryResponder> executeResponders = ComponentFactory.getExecuteResponders();
private List<SemanticCorrector> semanticCorrectors = ComponentFactory.getSqlCorrections();
@Override
@@ -159,21 +157,12 @@ public class QueryServiceImpl implements QueryService {
//5. postProcessor
postProcessors.forEach(postProcessor -> {
long startTime = System.currentTimeMillis();
postProcessor.process(queryCtx);
postProcessor.process(parseResult, queryCtx, chatCtx);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(postProcessor.getClass().getSimpleName())
.type(CostType.POSTPROCESSOR.getType()).build());
log.info("{} result:{}", postProcessor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
//6. responder
parseResponders.forEach(parseResponder -> {
long startTime = System.currentTimeMillis();
parseResponder.fillResponse(parseResult, queryCtx, chatCtx);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(parseResponder.getClass().getSimpleName())
.type(CostType.PARSERRESPONDER.getType()).build());
log.info("{} result:{}", parseResponder.getClass().getSimpleName(), JsonUtil.toString(parseResult));
});
if (Objects.nonNull(parseResult.getQueryId()) && timeCostDOList.size() > 0) {
saveTimeCostInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(),
@@ -217,7 +206,7 @@ public class QueryServiceImpl implements QueryService {
}
chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName());
for (ExecuteResponder executeResponder : executeResponders) {
for (QueryResponder executeResponder : executeResponders) {
executeResponder.fillResponse(queryResult, parseInfo, queryReq);
}
chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);

View File

@@ -7,8 +7,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.parser.LLMInterpreter;
import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
import com.tencent.supersonic.chat.postprocessor.PostProcessor;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.responder.QueryResponder;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.core.io.support.SpringFactoriesLoader;
import java.util.ArrayList;
@@ -24,8 +23,7 @@ public class ComponentFactory {
private static LLMInterpreter llmInterpreter;
private static List<PostProcessor> postProcessors = new ArrayList<>();
private static List<ParseResponder> parseResponders = new ArrayList<>();
private static List<ExecuteResponder> executeResponders = new ArrayList<>();
private static List<QueryResponder> executeResponders = new ArrayList<>();
private static ModelResolver modelResolver;
public static List<SchemaMapper> getSchemaMappers() {
@@ -45,13 +43,9 @@ public class ComponentFactory {
return CollectionUtils.isEmpty(postProcessors) ? init(PostProcessor.class, postProcessors) : postProcessors;
}
public static List<ParseResponder> getParseResponders() {
return CollectionUtils.isEmpty(parseResponders) ? init(ParseResponder.class, parseResponders) : parseResponders;
}
public static List<ExecuteResponder> getExecuteResponders() {
public static List<QueryResponder> getExecuteResponders() {
return CollectionUtils.isEmpty(executeResponders)
? init(ExecuteResponder.class, executeResponders) : executeResponders;
? init(QueryResponder.class, executeResponders) : executeResponders;
}
public static SemanticInterpreter getSemanticLayer() {

View File

@@ -34,9 +34,15 @@ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor
com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.SqlInfoParseResponder
com.tencent.supersonic.chat.postprocessor.PostProcessor=\
com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder
com.tencent.supersonic.chat.responder.QueryResponder=\
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder

View File

@@ -38,15 +38,13 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.chat.postprocessor.PostProcessor=\
com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.QueryRankParseResponder, \
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.SqlOutputParseResponder, \
com.tencent.supersonic.chat.responder.parse.TimeCostParseResponder, \
com.tencent.supersonic.chat.responder.parse.RespBuildParseResponder
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder, \
com.tencent.supersonic.chat.responder.execute.SimilarMetricExecuteResponder
com.tencent.supersonic.chat.responder.QueryResponder=\
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder

View File

@@ -34,15 +34,14 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.chat.postprocessor.PostProcessor=\
com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.QueryRankParseResponder, \
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.SqlOutputParseResponder, \
com.tencent.supersonic.chat.responder.parse.TimeCostParseResponder, \
com.tencent.supersonic.chat.responder.parse.RespBuildParseResponder
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder, \
com.tencent.supersonic.chat.responder.execute.SimilarMetricExecuteResponder
com.tencent.supersonic.chat.responder.QueryResponder=\
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder