From 4291ec7bd78c73bb8715506927fccccf9e442640 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 15 Mar 2024 16:08:34 +0800 Subject: [PATCH] (improvement)(Chat) Integrate chat with execute parse result processor (#825) Co-authored-by: jolunoluo --- .../api/pojo/request/ChatQueryDataReq.java | 22 +++ .../chat}/api/pojo/response/QueryResp.java | 4 +- .../chat/api/pojo/response/ShowCaseResp.java | 1 - .../pojo/response/SimilarQueryRecallResp.java | 2 - .../repository/ChatQueryRepository.java | 4 +- .../impl/ChatQueryRepositoryImpl.java | 17 ++- .../execute/DimensionRecommendProcessor.java | 17 ++- .../execute/ExecuteResultProcessor.java | 7 +- .../execute/MetricRatioProcessor.java | 26 ++-- .../execute/MetricRecommendProcessor.java | 7 +- .../parse/QueryRecommendProcessor.java | 9 +- .../chat/server/rest/ChatController.java | 2 +- .../chat/server/rest/ChatQueryController.java | 8 +- .../chat/server/service/ChatService.java | 12 +- .../server/service/impl/ChatServiceImpl.java | 50 +++--- .../chat/server/util/SimilarQueryManager.java | 142 ++++++++++++++++++ .../headless/api/pojo}/AggregateInfo.java | 5 +- .../headless/api/pojo}/MetricInfo.java | 5 +- .../api/pojo/response/QueryResult.java | 4 + .../src/main/resources/application-local.yaml | 2 +- 20 files changed, 265 insertions(+), 81 deletions(-) create mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatQueryDataReq.java rename {headless/api/src/main/java/com/tencent/supersonic/headless => chat/api/src/main/java/com/tencent/supersonic/chat}/api/pojo/response/QueryResp.java (69%) create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/util/SimilarQueryManager.java rename {chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response => headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo}/AggregateInfo.java (75%) rename {chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response => headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo}/MetricInfo.java (80%) diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatQueryDataReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatQueryDataReq.java new file mode 100644 index 000000000..965246ed3 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatQueryDataReq.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import lombok.Data; + +import java.util.HashSet; +import java.util.Set; + +@Data +public class ChatQueryDataReq { + private User user; + private Set metrics = new HashSet<>(); + private Set dimensions = new HashSet<>(); + private Set dimensionFilters = new HashSet<>(); + private Set metricFilters = new HashSet<>(); + private DateConf dateInfo; + private Long queryId; + private Integer parseId; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java similarity index 69% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java index 473b35d1a..4a5445dc1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java @@ -1,6 +1,7 @@ -package com.tencent.supersonic.headless.api.pojo.response; +package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import lombok.Data; import java.util.Date; import java.util.List; @@ -17,5 +18,6 @@ public class QueryResp { private String queryText; private QueryResult queryResult; private List parseInfos; + private List similarQueries; } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java index 1669b68f2..9b386e3c5 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; import lombok.Data; import java.util.List; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java index 19c8c0272..cde489190 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java @@ -10,8 +10,6 @@ public class SimilarQueryRecallResp { private Long queryId; - private Integer parseId; - private String queryText; } \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java index dda1aea8b..9f0d46c68 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import java.util.List; @@ -25,6 +25,8 @@ public interface ChatQueryRepository { int updateChatQuery(ChatQueryDO chatQueryDO); + Long createChatQuery(ChatParseReq chatParseReq); + List batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult, List candidateParses); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java index 77eccb1a2..f07f626c9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.chat.server.persistence.repository.impl; +import com.alibaba.fastjson.JSONObject; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDOExample; @@ -16,7 +18,7 @@ import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.PageUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -24,7 +26,6 @@ import org.springframework.beans.BeanUtils; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -106,10 +107,13 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { queryResult.setQueryId(chatQueryDO.getQuestionId()); queryResp.setQueryResult(queryResult); } + queryResp.setSimilarQueries(JSONObject.parseArray(chatQueryDO.getSimilarQueries(), + SimilarQueryRecallResp.class)); return queryResp; } - public Long createChatQuery(ParseResp parseResult, ChatParseReq chatParseReq) { + @Override + public Long createChatQuery(ChatParseReq chatParseReq) { ChatQueryDO chatQueryDO = new ChatQueryDO(); chatQueryDO.setChatId(Long.valueOf(chatParseReq.getChatId())); chatQueryDO.setCreateTime(new java.util.Date()); @@ -122,17 +126,14 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } catch (Exception e) { log.info("database insert has an exception:{}", e.toString()); } - Long queryId = chatQueryDO.getQuestionId(); - parseResult.setQueryId(queryId); - return queryId; + return chatQueryDO.getQuestionId(); } @Override public List batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult, List candidateParses) { - Long queryId = createChatQuery(parseResult, chatParseReq); List chatParseDOList = new ArrayList<>(); - getChatParseDO(chatParseReq, queryId, candidateParses, chatParseDOList); + getChatParseDO(chatParseReq, parseResult.getQueryId(), candidateParses, chatParseDOList); if (!CollectionUtils.isEmpty(candidateParses)) { chatParseMapper.batchSaveParseInfo(chatParseDOList); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index fd770dd5e..d4e4e80e2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -1,15 +1,15 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.server.service.impl.SemanticService; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.util.ContextUtils; import org.springframework.util.CollectionUtils; import java.util.Comparator; @@ -28,14 +28,15 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { private static final int recommend_dimension_size = 5; @Override - public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { + public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) { + SemanticParseInfo semanticParseInfo = chatExecuteContext.getParseInfo(); if (!QueryType.METRIC.equals(semanticParseInfo.getQueryType()) || CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) { return; } - //SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); - //List dimensionRecommended = getDimensions(element.getId(), element.getDataSet()); - //queryResult.setRecommendedDimensions(dimensionRecommended); + SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); + List dimensionRecommended = getDimensions(element.getId(), element.getDataSet()); + queryResult.setRecommendedDimensions(dimensionRecommended); } private List getDimensions(Long metricId, Long dataSetId) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java index 4108df8ad..e745263dd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java @@ -1,15 +1,14 @@ package com.tencent.supersonic.chat.server.processor.execute; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; import com.tencent.supersonic.chat.server.processor.ResultProcessor; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; /** * A ExecuteResultProcessor wraps things up before returning results to users in execute stage. */ public interface ExecuteResultProcessor extends ResultProcessor { - void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq); + void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult); } \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java index 20051b45b..76e66a639 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -1,8 +1,7 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; -import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; +import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.QueryColumn; @@ -11,14 +10,17 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.RatioOverType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.AggregateInfo; +import com.tencent.supersonic.headless.api.pojo.MetricInfo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.config.AggregatorConfig; import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; +import com.tencent.supersonic.headless.server.service.QueryService; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -56,19 +58,18 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK; @Slf4j public class MetricRatioProcessor implements ExecuteResultProcessor { - //private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - @Override - public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { - + public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) { + SemanticParseInfo semanticParseInfo = chatExecuteContext.getParseInfo(); AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class); if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio() || !QueryType.METRIC.equals(semanticParseInfo.getQueryType())) { return; } - //AggregateInfo aggregateInfo = getAggregateInfo(queryReq.getUser(), semanticParseInfo, queryResult); - //queryResult.setAggregateInfo(aggregateInfo); + AggregateInfo aggregateInfo = getAggregateInfo(chatExecuteContext.getUser(), + semanticParseInfo, queryResult); + queryResult.setAggregateInfo(aggregateInfo); } public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { @@ -123,16 +124,17 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { return aggregateInfo; } + @SneakyThrows private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum, QueryResult queryResult) { + AggOperatorEnum aggOperatorEnum, QueryResult queryResult) { QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField))); queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult)); queryStructReq.setConvertToSql(false); - - SemanticQueryResp queryResp = null; + QueryService queryService = ContextUtils.getBean(QueryService.class); + SemanticQueryResp queryResp = queryService.queryByReq(queryStructReq, user); MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); if (Objects.isNull(queryResp) || CollectionUtils.isEmpty(queryResp.getResultList())) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index cef7df02d..9cb49d6ce 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; @@ -10,9 +11,10 @@ import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.core.chat.knowledge.MetaEmbeddingService; +import org.springframework.util.CollectionUtils; + import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -21,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; /** * MetricRecommendProcessor fills recommended metrics based on embedding similarity. @@ -31,7 +32,7 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { private static final int METRIC_RECOMMEND_SIZE = 5; @Override - public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { + public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) { fillSimilarMetric(queryResult.getChatContext()); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java index 46c373a0d..4a43012e6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java @@ -8,9 +8,10 @@ import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.server.pojo.ChatParseContext; +import com.tencent.supersonic.chat.server.util.SimilarQueryManager; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -35,7 +36,7 @@ public class QueryRecommendProcessor implements ParseResultProcessor { private void doProcess(ParseResp parseResp, ChatParseContext chatParseContext) { Long queryId = parseResp.getQueryId(); List solvedQueries = getSimilarQueries(chatParseContext.getQueryText(), - null); + chatParseContext.getAgent().getId()); ChatQueryDO chatQueryDO = getChatQuery(queryId); chatQueryDO.setSimilarQueries(JSONObject.toJSONString(solvedQueries)); updateChatQuery(chatQueryDO); @@ -43,8 +44,8 @@ public class QueryRecommendProcessor implements ParseResultProcessor { public List getSimilarQueries(String queryText, Integer agentId) { //1. recall solved query by queryText - //SimilarQueryManager solvedQueryManager = ContextUtils.getBean(SimilarQueryManager.class); - List similarQueries = Lists.newArrayList(); + SimilarQueryManager solvedQueryManager = ContextUtils.getBean(SimilarQueryManager.class); + List similarQueries = solvedQueryManager.recallSimilarQuery(queryText, agentId); if (CollectionUtils.isEmpty(similarQueries)) { return Lists.newArrayList(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java index b4d2d8e51..687fbd7a1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.service.ChatService; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java index 4737caa31..52914259a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java @@ -4,9 +4,9 @@ package com.tencent.supersonic.chat.server.rest; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -58,10 +58,10 @@ public class ChatQueryController { } @PostMapping("queryData") - public Object queryData(@RequestBody QueryDataReq queryData, + public Object queryData(@RequestBody ChatQueryDataReq chatQueryDataReq, HttpServletRequest request, HttpServletResponse response) throws Exception { - queryData.setUser(UserHolder.findUser(request, response)); - return chatService.queryData(queryData, UserHolder.findUser(request, response)); + chatQueryDataReq.setUser(UserHolder.findUser(request, response)); + return chatService.queryData(chatQueryDataReq, UserHolder.findUser(request, response)); } @PostMapping("queryDimensionValue") diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java index b7ce33c0b..170ea1df1 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java @@ -4,16 +4,16 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; @@ -27,7 +27,7 @@ public interface ChatService { QueryResult performExecution(ChatExecuteReq chatExecuteReq) throws Exception; - Object queryData(QueryDataReq queryData, User user) throws Exception; + Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception; SemanticParseInfo queryContext(Integer chatId); @@ -53,13 +53,7 @@ public interface ChatService { List batchAddParse(ChatParseReq chatParseReq, ParseResp parseResult); - ChatQueryDO getLastQuery(long chatId); - int updateQuery(ChatQueryDO chatQueryDO); void saveQueryResult(ChatExecuteReq chatExecuteReq, QueryResult queryResult); - - ChatParseDO getParseInfo(Long questionId, int parseId); - - Boolean deleteChatQuery(Long questionId); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index d83aa25db..3fcd1fbcc 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -5,7 +5,9 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.executor.ChatExecutor; @@ -18,11 +20,13 @@ import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryReposi import com.tencent.supersonic.chat.server.persistence.repository.ChatRepository; import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; import com.tencent.supersonic.chat.server.pojo.ChatParseContext; +import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor; import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.util.ComponentFactory; import com.tencent.supersonic.chat.server.util.QueryReqConverter; +import com.tencent.supersonic.chat.server.util.SimilarQueryManager; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; @@ -32,7 +36,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.response.MapResp; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; import com.tencent.supersonic.headless.server.service.ChatQueryService; @@ -63,9 +67,12 @@ public class ChatServiceImpl implements ChatService { private ChatQueryService chatQueryService; @Autowired private SearchService searchService; + @Autowired + private SimilarQueryManager similarQueryManager; private List chatParsers = ComponentFactory.getChatParsers(); private List chatExecutors = ComponentFactory.getChatExecutors(); private List parseResultProcessors = ComponentFactory.getParseProcessors(); + private List executeResultProcessors = ComponentFactory.getExecuteProcessors(); @Override public List search(ChatParseReq chatParseReq) { @@ -77,6 +84,7 @@ public class ChatServiceImpl implements ChatService { @Override public ParseResp performParsing(ChatParseReq chatParseReq) { ParseResp parseResp = new ParseResp(chatParseReq.getChatId(), chatParseReq.getQueryText()); + createChatQuery(chatParseReq, parseResp); ChatParseContext chatParseContext = buildParseContext(chatParseReq); for (ChatParser chatParser : chatParsers) { chatParser.parse(chatParseContext, parseResp); @@ -98,6 +106,9 @@ public class ChatServiceImpl implements ChatService { break; } } + for (ExecuteResultProcessor processor : executeResultProcessors) { + processor.process(chatExecuteContext, queryResult); + } saveQueryResult(chatExecuteReq, queryResult); return queryResult; } @@ -117,15 +128,18 @@ public class ChatServiceImpl implements ChatService { private ChatExecuteContext buildExecuteContext(ChatExecuteReq chatExecuteReq) { ChatExecuteContext chatExecuteContext = new ChatExecuteContext(); BeanMapper.mapper(chatExecuteReq, chatExecuteContext); - ChatParseDO chatParseDO = getParseInfo(chatExecuteReq.getQueryId(), chatExecuteReq.getParseId()); - SemanticParseInfo semanticParseInfo = JSONObject.parseObject(chatParseDO.getParseInfo(), - SemanticParseInfo.class); - chatExecuteContext.setParseInfo(semanticParseInfo); + SemanticParseInfo parseInfo = getParseInfo(chatExecuteReq.getQueryId(), chatExecuteReq.getParseId()); + chatExecuteContext.setParseInfo(parseInfo); return chatExecuteContext; } @Override - public Object queryData(QueryDataReq queryData, User user) throws Exception { + public Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception { + Integer parseId = chatQueryDataReq.getParseId(); + SemanticParseInfo parseInfo = getParseInfo(chatQueryDataReq.getQueryId(), parseId); + QueryDataReq queryData = new QueryDataReq(); + BeanMapper.mapper(chatQueryDataReq, queryData); + queryData.setParseInfo(parseInfo); return chatQueryService.executeDirectQuery(queryData, user); } @@ -192,6 +206,11 @@ public class ChatServiceImpl implements ChatService { return queryRespPageInfo; } + public void createChatQuery(ChatParseReq chatParseReq, ParseResp parseResp) { + Long queryId = chatQueryRepository.createChatQuery(chatParseReq); + parseResp.setQueryId(queryId); + } + @Override public QueryResp getChatQuery(Long queryId) { return chatQueryRepository.getChatQuery(queryId); @@ -257,13 +276,16 @@ public class ChatServiceImpl implements ChatService { if (chatExecuteReq.getParseId() > 1) { return; } - ChatQueryDO chatQueryDO = new ChatQueryDO(); + ChatQueryDO chatQueryDO = chatQueryRepository.getChatQueryDO(chatExecuteReq.getQueryId()); chatQueryDO.setQuestionId(chatExecuteReq.getQueryId()); chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); chatQueryDO.setQueryState(1); updateQuery(chatQueryDO); chatRepository.updateLastQuestion(chatExecuteReq.getChatId().longValue(), chatExecuteReq.getQueryText(), getCurrentTime()); + SimilarQueryReq similarQueryReq = SimilarQueryReq.builder().queryId(chatExecuteReq.getQueryId()) + .queryText(chatQueryDO.getQueryText()).agentId(chatQueryDO.getAgentId()).build(); + similarQueryManager.saveSimilarQuery(similarQueryReq); } @Override @@ -277,22 +299,14 @@ public class ChatServiceImpl implements ChatService { return chatQueryRepository.batchSaveParseInfo(chatParseReq, parseResult, candidateParses); } - @Override - public ChatQueryDO getLastQuery(long chatId) { - return chatQueryRepository.getLastChatQuery(chatId); - } - private String getCurrentTime() { SimpleDateFormat tempDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return tempDate.format(new java.util.Date()); } - public ChatParseDO getParseInfo(Long questionId, int parseId) { - return chatQueryRepository.getParseInfo(questionId, parseId); - } - - public Boolean deleteChatQuery(Long questionId) { - return chatQueryRepository.deleteChatQuery(questionId); + public SemanticParseInfo getParseInfo(Long questionId, int parseId) { + ChatParseDO chatParseDO = chatQueryRepository.getParseInfo(questionId, parseId); + return JSONObject.parseObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/SimilarQueryManager.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/SimilarQueryManager.java new file mode 100644 index 000000000..adaa78904 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/SimilarQueryManager.java @@ -0,0 +1,142 @@ +package com.tencent.supersonic.chat.server.util; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; +import com.tencent.supersonic.common.config.EmbeddingConfig; +import com.tencent.supersonic.common.util.ComponentFactory; +import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; +import com.tencent.supersonic.common.util.embedding.Retrieval; +import com.tencent.supersonic.common.util.embedding.RetrieveQuery; +import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; +import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class SimilarQueryManager { + + private EmbeddingConfig embeddingConfig; + + private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); + + + public SimilarQueryManager(EmbeddingConfig embeddingConfig) { + this.embeddingConfig = embeddingConfig; + } + + public void saveSimilarQuery(SimilarQueryReq similarQueryReq) { + if (StringUtils.isBlank(embeddingConfig.getUrl())) { + return; + } + String queryText = similarQueryReq.getQueryText(); + try { + EmbeddingQuery embeddingQuery = new EmbeddingQuery(); + embeddingQuery.setQueryId(String.valueOf(similarQueryReq.getQueryId())); + embeddingQuery.setQuery(queryText); + + Map metaData = new HashMap<>(); + metaData.put("agentId", similarQueryReq.getAgentId()); + embeddingQuery.setMetadata(metaData); + String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); + s2EmbeddingStore.addQuery(solvedQueryCollection, Lists.newArrayList(embeddingQuery)); + } catch (Exception e) { + log.warn("save history question to embedding failed, queryText:{}", queryText, e); + } + } + + public List recallSimilarQuery(String queryText, Integer agentId) { + if (StringUtils.isBlank(embeddingConfig.getUrl())) { + return Lists.newArrayList(); + } + List similarQueryRecallResps = Lists.newArrayList(); + try { + String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); + int solvedQueryResultNum = embeddingConfig.getSolvedQueryResultNum(); + + Map filterCondition = new HashMap<>(); + filterCondition.put("agentId", String.valueOf(agentId)); + RetrieveQuery retrieveQuery = RetrieveQuery.builder() + .queryTextsList(Lists.newArrayList(queryText)) + .filterCondition(filterCondition) + .build(); + List resultList = s2EmbeddingStore.retrieveQuery(solvedQueryCollection, retrieveQuery, + solvedQueryResultNum * 20); + + log.info("[embedding] recognize result body:{}", resultList); + Set querySet = new HashSet<>(); + if (CollectionUtils.isNotEmpty(resultList)) { + for (RetrieveQueryResult retrieveQueryResult : resultList) { + List retrievals = retrieveQueryResult.getRetrieval(); + for (Retrieval retrieval : retrievals) { + if (queryText.equalsIgnoreCase(retrieval.getQuery())) { + continue; + } + if (querySet.contains(retrieval.getQuery())) { + continue; + } + String id = retrieval.getId(); + SimilarQueryRecallResp similarQueryRecallResp = SimilarQueryRecallResp.builder() + .queryText(retrieval.getQuery()) + .queryId(Long.parseLong(id)) + .build(); + similarQueryRecallResps.add(similarQueryRecallResp); + querySet.add(retrieval.getQuery()); + } + } + } + + } catch (Exception e) { + log.warn("recall similar solved query failed, queryText:{}", queryText, e); + } + return similarQueryRecallResps.stream() + .limit(embeddingConfig.getSolvedQueryResultNum()).collect(Collectors.toList()); + } + + private ResponseEntity doRequest(String path, String jsonBody) { + if (Strings.isEmpty(embeddingConfig.getUrl())) { + return ResponseEntity.of(Optional.empty()); + } + String url = embeddingConfig.getUrl() + path; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setLocation(URI.create(url)); + URI requestUrl = UriComponentsBuilder + .fromHttpUrl(url).build().encode().toUri(); + HttpEntity entity = new HttpEntity<>(jsonBody, headers); + log.info("[embedding] request body :{}, url:{}", jsonBody, url); + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity responseEntity = restTemplate.exchange(requestUrl, + HttpMethod.POST, entity, new ParameterizedTypeReference() { + }); + log.info("[embedding] result body:{}", responseEntity); + return responseEntity; + } catch (Exception e) { + log.warn("connect to embedding service failed, url:{}", url); + } + return ResponseEntity.of(Optional.empty()); + } + +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/AggregateInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AggregateInfo.java similarity index 75% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/AggregateInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AggregateInfo.java index aebbeb1b0..24dbf1f71 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/AggregateInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AggregateInfo.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; import java.util.ArrayList; import java.util.List; -import lombok.Data; @Data public class AggregateInfo { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java similarity index 80% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java index 1c1c4c580..299b710fc 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricInfo.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; import java.util.Map; -import lombok.Data; @Data public class MetricInfo { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java index b85570c27..75cab62a6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java @@ -2,7 +2,9 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.headless.api.pojo.AggregateInfo; import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; @@ -22,4 +24,6 @@ public class QueryResult { private List> queryResults; private Long queryTimeCost; private EntityInfo entityInfo; + private List recommendedDimensions; + private AggregateInfo aggregateInfo; } diff --git a/launchers/standalone/src/main/resources/application-local.yaml b/launchers/standalone/src/main/resources/application-local.yaml index 1acba84ae..cbe243794 100644 --- a/launchers/standalone/src/main/resources/application-local.yaml +++ b/launchers/standalone/src/main/resources/application-local.yaml @@ -95,7 +95,7 @@ logging: inMemoryEmbeddingStore: persistent: - path: /tmp + path: d:// query: optimizer: