diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java similarity index 99% rename from auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java rename to auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java index 1a36d2352..8496e1f4a 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authorization.application; +package com.tencent.supersonic.auth.authorization.service; import com.google.common.base.Strings; import com.google.common.collect.Lists; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java similarity index 92% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java index 56506b48a..3ce7e3d1f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java @@ -10,7 +10,7 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @AllArgsConstructor @Builder -public class SolvedQueryReq { +public class SimilarQueryReq { private Long queryId; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java index f5d764336..517f24b6f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java @@ -6,6 +6,6 @@ import java.util.List; @Data public class QueryRecallResp { - private List solvedQueryRecallRespList; + private List solvedQueryRecallRespList; private Long queryTimeCost; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java index 535841552..fa75042cb 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.api.pojo.response; -import java.util.Date; -import java.util.List; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import lombok.Data; +import java.util.Date; +import java.util.List; @Data public class QueryResp { @@ -16,4 +16,5 @@ public class QueryResp { private String queryText; private QueryResult queryResult; private List parseInfos; + private List similarQueries; } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java index 093f7e0c1..44e5ec691 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.chat.api.pojo.response; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; +import lombok.Data; import java.util.List; import java.util.Map; -import lombok.Data; @Data public class QueryResult { @@ -22,4 +23,5 @@ public class QueryResult { private Object response; private List> queryResults; private Long queryTimeCost; + private List recommendedDimensions; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SolvedQueryRecallResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java similarity index 84% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SolvedQueryRecallResp.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java index e92dd1cb1..19c8c0272 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SolvedQueryRecallResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SimilarQueryRecallResp.java @@ -6,7 +6,7 @@ import lombok.Data; @Data @Builder -public class SolvedQueryRecallResp { +public class SimilarQueryRecallResp { private Long queryId; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatQueryDO.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatQueryDO.java index 6d5a8ec6e..663b1532e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatQueryDO.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatQueryDO.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.chat.persistence.dataobject; +import lombok.Data; import java.util.Date; +@Data public class ChatQueryDO { /** */ @@ -43,155 +45,6 @@ public class ChatQueryDO { */ private String queryResult; - /** - * @return question_id - */ - public Long getQuestionId() { - return questionId; - } + private String similarQueries; - /** - * @param questionId - */ - public void setQuestionId(Long questionId) { - this.questionId = questionId; - } - - /** - * @return agent_id - */ - public Integer getAgentId() { - return agentId; - } - - /** - * @param agentId - */ - public void setAgentId(Integer agentId) { - this.agentId = agentId; - } - - /** - * @return create_time - */ - public Date getCreateTime() { - return createTime; - } - - /** - * @param createTime - */ - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - /** - * @return user_name - */ - public String getUserName() { - return userName; - } - - /** - * @param userName - */ - public void setUserName(String userName) { - this.userName = userName == null ? null : userName.trim(); - } - - /** - * - * @return query_state - */ - public Integer getQueryState() { - return queryState; - } - - /** - * - * @param queryState - */ - public void setQueryState(Integer queryState) { - this.queryState = queryState; - } - - /** - * - * @return chat_id - */ - public Long getChatId() { - return chatId; - } - - /** - * - * @param chatId - */ - public void setChatId(Long chatId) { - this.chatId = chatId; - } - - /** - * - * @return score - */ - public Integer getScore() { - return score; - } - - /** - * - * @param score - */ - public void setScore(Integer score) { - this.score = score; - } - - /** - * - * @return feedback - */ - public String getFeedback() { - return feedback; - } - - /** - * - * @param feedback - */ - public void setFeedback(String feedback) { - this.feedback = feedback == null ? null : feedback.trim(); - } - - /** - * - * @return query_text - */ - public String getQueryText() { - return queryText; - } - - /** - * - * @param queryText - */ - public void setQueryText(String queryText) { - this.queryText = queryText == null ? null : queryText.trim(); - } - - /** - * - * @return query_result - */ - public String getQueryResult() { - return queryResult; - } - - /** - * - * @param queryResult - */ - public void setQueryResult(String queryResult) { - this.queryResult = queryResult == null ? null : queryResult.trim(); - } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatQueryDOMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatQueryDOMapper.java index dbc1c332f..9f3e5d766 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatQueryDOMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatQueryDOMapper.java @@ -16,4 +16,6 @@ public interface ChatQueryDOMapper { int updateByPrimaryKeyWithBLOBs(ChatQueryDO record); Boolean deleteByPrimaryKey(Long questionId); + + ChatQueryDO selectByPrimaryKey(Long questionId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java index 8dc8cf33d..e716871da 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java @@ -3,13 +3,12 @@ package com.tencent.supersonic.chat.persistence.repository; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import java.util.List; @@ -17,9 +16,11 @@ public interface ChatQueryRepository { PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoCommend, Long chatId); - List queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId); + QueryResp getChatQuery(Long queryId); - void createChatQuery(QueryResult queryResult, ChatContext chatCtx); + ChatQueryDO getChatQueryDO(Long queryId); + + List queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId); void updateChatParseInfo(List chatParseDOS); @@ -27,13 +28,11 @@ public interface ChatQueryRepository { int updateChatQuery(ChatQueryDO chatQueryDO); - Long createChatParse(ParseResp parseResult, ChatContext chatCtx, QueryReq queryReq); - List batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq, ParseResp parseResult, List candidateParses); - public ChatParseDO getParseInfo(Long questionId, int parseId); + ChatParseDO getParseInfo(Long questionId, int parseId); List getParseInfoList(List questionIds); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java index 521ee426f..c653c388b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.chat.persistence.repository.impl; +import com.alibaba.fastjson.JSONObject; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.ChatContext; @@ -9,6 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample; @@ -28,6 +30,7 @@ import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; @Repository @@ -75,6 +78,20 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { return chatQueryVOPageInfo; } + @Override + public QueryResp getChatQuery(Long queryId) { + ChatQueryDO chatQueryDO = getChatQueryDO(queryId); + if (Objects.isNull(chatQueryDO)) { + return new QueryResp(); + } + return convertTo(chatQueryDO); + } + + @Override + public ChatQueryDO getChatQueryDO(Long queryId) { + return chatQueryDOMapper.selectByPrimaryKey(queryId); + } + @Override public List queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId) { return showCaseCustomMapper.queryShowCase(pageQueryInfoReq.getLimitStart(), @@ -84,33 +101,22 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } private QueryResp convertTo(ChatQueryDO chatQueryDO) { - QueryResp queryResponse = new QueryResp(); - BeanUtils.copyProperties(chatQueryDO, queryResponse); + QueryResp queryResp = new QueryResp(); + BeanUtils.copyProperties(chatQueryDO, queryResp); QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class); if (queryResult != null) { queryResult.setQueryId(chatQueryDO.getQuestionId()); - queryResponse.setQueryResult(queryResult); + queryResp.setQueryResult(queryResult); } - return queryResponse; + if (StringUtils.isNotBlank(chatQueryDO.getSimilarQueries())) { + List similarQueries = JSONObject.parseArray(chatQueryDO.getSimilarQueries(), + SimilarQueryRecallResp.class); + queryResp.setSimilarQueries(similarQueries); + } + return queryResp; } - @Override - public void createChatQuery(QueryResult queryResult, ChatContext chatCtx) { - ChatQueryDO chatQueryDO = new ChatQueryDO(); - chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId())); - chatQueryDO.setCreateTime(new java.util.Date()); - chatQueryDO.setUserName(chatCtx.getUser()); - chatQueryDO.setQueryState(queryResult.getQueryState().ordinal()); - chatQueryDO.setQueryText(chatCtx.getQueryText()); - chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); - chatQueryDO.setAgentId(chatCtx.getAgentId()); - chatQueryDOMapper.insert(chatQueryDO); - ChatQueryDO lastChatQuery = getLastChatQuery(chatCtx.getChatId()); - Long queryId = lastChatQuery.getQuestionId(); - queryResult.setQueryId(queryId); - } - - public Long createChatParse(ParseResp parseResult, ChatContext chatCtx, QueryReq queryReq) { + public Long createChatQuery(ParseResp parseResult, ChatContext chatCtx, QueryReq queryReq) { ChatQueryDO chatQueryDO = new ChatQueryDO(); chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId())); chatQueryDO.setCreateTime(new java.util.Date()); @@ -132,7 +138,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { public List batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq, ParseResp parseResult, List candidateParses) { - Long queryId = createChatParse(parseResult, chatCtx, queryReq); + Long queryId = createChatQuery(parseResult, chatCtx, queryReq); List chatParseDOList = new ArrayList<>(); getChatParseDO(chatCtx, queryReq, queryId, candidateParses, chatParseDOList); chatParseMapper.batchSaveParseInfo(chatParseDOList); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DimensionRecommendProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DimensionRecommendProcessor.java new file mode 100644 index 000000000..72b5ce035 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DimensionRecommendProcessor.java @@ -0,0 +1,73 @@ +package com.tencent.supersonic.chat.processor.execute; + +import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.service.SemanticService; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.util.ContextUtils; +import org.apache.commons.compress.utils.Lists; +import org.springframework.util.CollectionUtils; + +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * DrillDownDimensionProcessor obtains metric recommended dimensions + */ +public class DimensionRecommendProcessor implements ExecuteResultProcessor { + + private static final int recommend_dimension_size = 5; + + @Override + public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { + if (!QueryType.METRIC.equals(semanticParseInfo.getQueryType()) + || CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) { + return; + } + SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); + List dimensionRecommended = getDimensions(element.getId(), element.getModel()); + queryResult.setRecommendedDimensions(dimensionRecommended); + } + + private List getDimensions(Long metricId, Long modelId) { + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + ModelSchema modelSchema = semanticService.getModelSchema(modelId); + List drillDownDimensions = Lists.newArrayList(); + Set metricElements = modelSchema.getMetrics(); + if (!CollectionUtils.isEmpty(metricElements)) { + Optional metric = metricElements.stream().filter(schemaElement -> + metricId.equals(schemaElement.getId()) + && !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements())) + .findFirst(); + if (metric.isPresent()) { + drillDownDimensions = metric.get().getRelatedSchemaElements().stream() + .map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()); + } + } + final List drillDownDimensionsFinal = drillDownDimensions; + return modelSchema.getDimensions().stream() + .filter(dim -> filterDimension(drillDownDimensionsFinal, dim)) + .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(recommend_dimension_size) + .collect(Collectors.toList()); + } + + private boolean filterDimension(List drillDownDimensions, SchemaElement dimension) { + if (Objects.isNull(dimension)) { + return false; + } + if (!CollectionUtils.isEmpty(drillDownDimensions)) { + return drillDownDimensions.contains(dimension.getId()); + } + return Objects.nonNull(dimension.getUseCnt()); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DrillDownDimensionProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DrillDownDimensionProcessor.java new file mode 100644 index 000000000..e10b01854 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/execute/DrillDownDimensionProcessor.java @@ -0,0 +1,73 @@ +package com.tencent.supersonic.chat.processor.execute; + +import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.service.SemanticService; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.util.ContextUtils; +import org.apache.commons.compress.utils.Lists; +import org.springframework.util.CollectionUtils; + +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * DrillDownDimensionProcessor obtains metric recommended dimensions based on setting + */ +public class DrillDownDimensionProcessor implements ExecuteResultProcessor { + + private static final int recommend_dimension_size = 5; + + @Override + public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { + if (!QueryType.METRIC.equals(semanticParseInfo.getQueryType()) + || CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) { + return; + } + SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); + List dimensionRecommend = getDimensions(element.getId(), element.getModel()); + queryResult.setRecommendDimensions(dimensionRecommend); + } + + private List getDimensions(Long metricId, Long modelId) { + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + ModelSchema modelSchema = semanticService.getModelSchema(modelId); + List drillDownDimensions = Lists.newArrayList(); + Set metricElements = modelSchema.getMetrics(); + if (!CollectionUtils.isEmpty(metricElements)) { + Optional metric = metricElements.stream().filter(schemaElement -> + metricId.equals(schemaElement.getId()) + && !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements())) + .findFirst(); + if (metric.isPresent()) { + drillDownDimensions = metric.get().getRelatedSchemaElements().stream() + .map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()); + } + } + final List drillDownDimensionsFinal = drillDownDimensions; + return modelSchema.getDimensions().stream() + .filter(dim -> filterDimension(drillDownDimensionsFinal, dim)) + .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(recommend_dimension_size) + .collect(Collectors.toList()); + } + + private boolean filterDimension(List drillDownDimensions, SchemaElement dimension) { + if (Objects.isNull(dimension)) { + return false; + } + if (!CollectionUtils.isEmpty(drillDownDimensions)) { + return drillDownDimensions.contains(dimension.getId()); + } + return Objects.nonNull(dimension.getUseCnt()); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SimilarQueryProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SimilarQueryProcessor.java new file mode 100644 index 000000000..06c917f35 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SimilarQueryProcessor.java @@ -0,0 +1,86 @@ +package com.tencent.supersonic.chat.processor.parse; + +import com.alibaba.fastjson.JSONObject; +import com.github.pagehelper.PageInfo; +import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; +import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; +import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; +import com.tencent.supersonic.chat.utils.SimilarQueryManager; +import com.tencent.supersonic.common.util.ContextUtils; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +@Slf4j +public class SimilarQueryProcessor implements ParseResultProcessor { + + @Override + public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { + CompletableFuture.runAsync(() -> doProcess(parseResp, queryContext)); + } + + @SneakyThrows + private void doProcess(ParseResp parseResp, QueryContext queryContext) { + Long queryId = parseResp.getQueryId(); + List solvedQueries = getSimilarQueries(queryContext.getRequest().getQueryText(), + queryContext.getRequest().getAgentId()); + ChatQueryDO chatQueryDO = getChatQuery(queryId); + chatQueryDO.setSimilarQueries(JSONObject.toJSONString(solvedQueries)); + updateChatQuery(chatQueryDO); + } + + public List getSimilarQueries(String queryText, Integer agentId) { + //1. recall solved query by queryText + SimilarQueryManager solvedQueryManager = ContextUtils.getBean(SimilarQueryManager.class); + List similarQueries = solvedQueryManager.recallSimilarQuery(queryText, agentId); + if (CollectionUtils.isEmpty(similarQueries)) { + return Lists.newArrayList(); + } + //2. remove low score query + List queryIds = similarQueries.stream() + .map(SimilarQueryRecallResp::getQueryId).collect(Collectors.toList()); + int lowScoreThreshold = 3; + List queryResps = getChatQuery(queryIds); + if (CollectionUtils.isEmpty(queryResps)) { + return Lists.newArrayList(); + } + Set lowScoreQueryIds = queryResps.stream().filter(queryResp -> + queryResp.getScore() != null && queryResp.getScore() <= lowScoreThreshold) + .map(QueryResp::getQuestionId).collect(Collectors.toSet()); + return similarQueries.stream().filter(solvedQueryRecallResp -> + !lowScoreQueryIds.contains(solvedQueryRecallResp.getQueryId())) + .collect(Collectors.toList()); + } + + private ChatQueryDO getChatQuery(Long queryId) { + ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class); + return chatQueryRepository.getChatQueryDO(queryId); + } + + private List getChatQuery(List queryIds) { + ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class); + PageQueryInfoReq pageQueryInfoReq = new PageQueryInfoReq(); + pageQueryInfoReq.setIds(queryIds); + pageQueryInfoReq.setPageSize(100); + pageQueryInfoReq.setCurrent(1); + PageInfo queryRespPageInfo = chatQueryRepository.getChatQuery(pageQueryInfoReq, null); + return queryRespPageInfo.getList(); + } + + private void updateChatQuery(ChatQueryDO chatQueryDO) { + ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class); + chatQueryRepository.updateChatQuery(chatQueryDO); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java index 1cce6735a..0b68d8f2d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java @@ -3,22 +3,23 @@ package com.tencent.supersonic.chat.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.api.pojo.response.QueryRecallResp; -import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; -import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; -import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.QueryRecallResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; +import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.service.ChatService; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; @RestController @RequestMapping({"/api/chat/manage", "/openapi/chat/manage"}) @@ -81,6 +82,11 @@ public class ChatController { return chatService.queryInfo(pageQueryInfoCommand, chatId); } + @GetMapping("/getChatQuery/{queryId}") + public QueryResp getChatQuery(@PathVariable("queryId") Long queryId) { + return chatService.getChatQuery(queryId); + } + @PostMapping("/queryShowCase") public ShowCaseResp queryShowCase(@RequestBody PageQueryInfoReq pageQueryInfoCommand, @RequestParam(value = "agentId") int agentId) { @@ -88,11 +94,11 @@ public class ChatController { } @RequestMapping("/getSolvedQuery") - public List getSolvedQuery(@RequestParam(value = "queryText") String queryText, - @RequestParam(value = "agentId") Integer agentId) { + public List getSolvedQuery(@RequestParam(value = "queryText") String queryText, + @RequestParam(value = "agentId") Integer agentId) { QueryRecallResp queryRecallResp = new QueryRecallResp(); Long startTime = System.currentTimeMillis(); - List solvedQueryRecallRespList = chatService.getSolvedQuery(queryText, agentId); + List solvedQueryRecallRespList = chatService.getSolvedQuery(queryText, agentId); queryRecallResp.setSolvedQueryRecallRespList(solvedQueryRecallRespList); queryRecallResp.setQueryTimeCost(System.currentTimeMillis() - startTime); return solvedQueryRecallRespList; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java index 1921e0a55..757ecd21f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; -import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; @@ -29,8 +29,6 @@ public interface ChatService { void updateContext(ChatContext chatCtx); - void switchContext(ChatContext chatCtx); - Boolean addChat(User user, String chatName, Integer agentId); List getAll(String userName, Integer agentId); @@ -45,14 +43,12 @@ public interface ChatService { PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId); + QueryResp getChatQuery(Long queryId); + ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId); - void addQuery(QueryResult queryResult, ChatContext chatCtx); - List batchAddParse(ChatContext chatCtx, QueryReq queryReq, ParseResp parseResult); - void updateChatParse(List chatParseDOS); - ChatQueryDO getLastQuery(long chatId); int updateQuery(ChatQueryDO chatQueryDO); @@ -63,5 +59,5 @@ public interface ChatService { Boolean deleteChatQuery(Long questionId); - List getSolvedQuery(String queryText, Integer agentId); + List getSolvedQuery(String queryText, Integer agentId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java index c801585bf..69b181a45 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; -import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; @@ -19,7 +19,7 @@ import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.persistence.repository.ChatRepository; import com.tencent.supersonic.chat.service.ChatService; -import com.tencent.supersonic.chat.utils.SolvedQueryManager; +import com.tencent.supersonic.chat.utils.SimilarQueryManager; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.utils.Lists; @@ -46,10 +46,10 @@ public class ChatServiceImpl implements ChatService { private ChatContextRepository chatContextRepository; private ChatRepository chatRepository; private ChatQueryRepository chatQueryRepository; - private SolvedQueryManager solvedQueryManager; + private SimilarQueryManager solvedQueryManager; public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository, - ChatQueryRepository chatQueryRepository, SolvedQueryManager solvedQueryManager) { + ChatQueryRepository chatQueryRepository, SimilarQueryManager solvedQueryManager) { this.chatContextRepository = chatContextRepository; this.chatRepository = chatRepository; this.chatQueryRepository = chatQueryRepository; @@ -83,12 +83,6 @@ public class ChatServiceImpl implements ChatService { chatContextRepository.updateContext(chatCtx); } - @Override - public void switchContext(ChatContext chatCtx) { - log.debug("switchContext ChatContext {}", chatCtx); - chatCtx.setParseInfo(new SemanticParseInfo()); - } - @Override public Boolean addChat(User user, String chatName, Integer agentId) { ChatDO chatDO = new ChatDO(); @@ -142,6 +136,11 @@ public class ChatServiceImpl implements ChatService { return queryRespPageInfo; } + @Override + public QueryResp getChatQuery(Long queryId) { + return chatQueryRepository.getChatQuery(queryId); + } + @Override public ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId) { ShowCaseResp showCaseResp = new ShowCaseResp(); @@ -196,13 +195,6 @@ public class ChatServiceImpl implements ChatService { } } - @Override - public void addQuery(QueryResult queryResult, ChatContext chatCtx) { - chatQueryRepository.createChatQuery(queryResult, chatCtx); - chatRepository.updateLastQuestion(chatCtx.getChatId().longValue(), - chatCtx.getQueryText(), getCurrentTime()); - } - @Override public Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx) { ChatQueryDO chatQueryDO = new ChatQueryDO(); @@ -226,11 +218,6 @@ public class ChatServiceImpl implements ChatService { return chatQueryRepository.batchSaveParseInfo(chatCtx, queryReq, parseResult, candidateParses); } - @Override - public void updateChatParse(List chatParseDOS) { - chatQueryRepository.updateChatParseInfo(chatParseDOS); - } - @Override public ChatQueryDO getLastQuery(long chatId) { return chatQueryRepository.getLastChatQuery(chatId); @@ -250,14 +237,14 @@ public class ChatServiceImpl implements ChatService { } @Override - public List getSolvedQuery(String queryText, Integer agentId) { + public List getSolvedQuery(String queryText, Integer agentId) { //1. recall solved query by queryText - List solvedQueryRecallResps = solvedQueryManager.recallSolvedQuery(queryText, agentId); + List solvedQueryRecallResps = solvedQueryManager.recallSimilarQuery(queryText, agentId); if (CollectionUtils.isEmpty(solvedQueryRecallResps)) { return Lists.newArrayList(); } List queryIds = solvedQueryRecallResps.stream() - .map(SolvedQueryRecallResp::getQueryId).collect(Collectors.toList()); + .map(SimilarQueryRecallResp::getQueryId).collect(Collectors.toList()); PageQueryInfoReq pageQueryInfoReq = new PageQueryInfoReq(); pageQueryInfoReq.setIds(queryIds); pageQueryInfoReq.setPageSize(100); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 2f6cabc43..21f083b69 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -19,7 +19,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; @@ -39,7 +39,7 @@ import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.StatisticsService; import com.tencent.supersonic.chat.service.TimeCost; import com.tencent.supersonic.chat.utils.ComponentFactory; -import com.tencent.supersonic.chat.utils.SolvedQueryManager; +import com.tencent.supersonic.chat.utils.SimilarQueryManager; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.QueryType; @@ -104,7 +104,7 @@ public class QueryServiceImpl implements QueryService { @Autowired private StatisticsService statisticsService; @Autowired - private SolvedQueryManager solvedQueryManager; + private SimilarQueryManager similarQueryManager; @Value("${time.threshold: 100}") private Integer timeThreshold; @@ -240,7 +240,7 @@ public class QueryServiceImpl implements QueryService { if (queryResult.getResponse() == null && CollectionUtils.isEmpty(queryResult.getQueryResults())) { return; } - solvedQueryManager.saveSolvedQuery(SolvedQueryReq.builder().parseId(queryReq.getParseId()) + similarQueryManager.saveSimilarQuery(SimilarQueryReq.builder().parseId(queryReq.getParseId()) .queryId(queryReq.getQueryId()) .agentId(chatQueryDO.getAgentId()) .modelId(parseInfo.getModelClusterKey()) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SimilarQueryManager.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/utils/SimilarQueryManager.java index 628e5db5a..ca7922267 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SimilarQueryManager.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.utils; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; +import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.ComponentFactory; import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; @@ -10,13 +10,6 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; -import java.net.URI; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -31,33 +24,41 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + @Slf4j @Component -public class SolvedQueryManager { +public class SimilarQueryManager { private EmbeddingConfig embeddingConfig; private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); - public SolvedQueryManager(EmbeddingConfig embeddingConfig) { + public SimilarQueryManager(EmbeddingConfig embeddingConfig) { this.embeddingConfig = embeddingConfig; } - public void saveSolvedQuery(SolvedQueryReq solvedQueryReq) { + public void saveSimilarQuery(SimilarQueryReq similarQueryReq) { if (StringUtils.isBlank(embeddingConfig.getUrl())) { return; } - String queryText = solvedQueryReq.getQueryText(); + String queryText = similarQueryReq.getQueryText(); try { - String uniqueId = generateUniqueId(solvedQueryReq.getQueryId(), solvedQueryReq.getParseId()); + String uniqueId = generateUniqueId(similarQueryReq.getQueryId(), similarQueryReq.getParseId()); EmbeddingQuery embeddingQuery = new EmbeddingQuery(); embeddingQuery.setQueryId(uniqueId); embeddingQuery.setQuery(queryText); Map metaData = new HashMap<>(); - metaData.put("modelId", (solvedQueryReq.getModelId())); - metaData.put("agentId", solvedQueryReq.getAgentId()); + metaData.put("modelId", (similarQueryReq.getModelId())); + metaData.put("agentId", similarQueryReq.getAgentId()); embeddingQuery.setMetadata(metaData); String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); s2EmbeddingStore.addQuery(solvedQueryCollection, Lists.newArrayList(embeddingQuery)); @@ -66,11 +67,11 @@ public class SolvedQueryManager { } } - public List recallSolvedQuery(String queryText, Integer agentId) { + public List recallSimilarQuery(String queryText, Integer agentId) { if (StringUtils.isBlank(embeddingConfig.getUrl())) { return Lists.newArrayList(); } - List solvedQueryRecallResps = Lists.newArrayList(); + List similarQueryRecallResps = Lists.newArrayList(); try { String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); int solvedQueryResultNum = embeddingConfig.getSolvedQueryResultNum(); @@ -97,11 +98,11 @@ public class SolvedQueryManager { continue; } String id = retrieval.getId(); - SolvedQueryRecallResp solvedQueryRecallResp = SolvedQueryRecallResp.builder() + SimilarQueryRecallResp similarQueryRecallResp = SimilarQueryRecallResp.builder() .queryText(retrieval.getQuery()) .queryId(getQueryId(id)).parseId(getParseId(id)) .build(); - solvedQueryRecallResps.add(solvedQueryRecallResp); + similarQueryRecallResps.add(similarQueryRecallResp); querySet.add(retrieval.getQuery()); } } @@ -110,7 +111,7 @@ public class SolvedQueryManager { } catch (Exception e) { log.warn("recall similar solved query failed, queryText:{}", queryText); } - return solvedQueryRecallResps; + return similarQueryRecallResps; } private String generateUniqueId(Long queryId, Integer parseId) { diff --git a/chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml b/chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml index 2c4554d26..3baaefe6d 100644 --- a/chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml +++ b/chat/core/src/main/resources/mapper/ChatQueryDOMapper.xml @@ -14,6 +14,7 @@ + @@ -48,7 +49,7 @@ question_id, agent_id, create_time, user_name, query_state, chat_id, score, feedback - query_text, query_result + query_text, query_result, similar_queries + + delete from s2_chat_query @@ -75,11 +81,12 @@ insert into s2_chat_query (agent_id, create_time, user_name, query_state, chat_id, score, - feedback, query_text, query_result + feedback, query_text, query_result, similar_queries ) values (#{agentId,jdbcType=INTEGER}, #{createTime,jdbcType=TIMESTAMP}, #{userName,jdbcType=VARCHAR}, #{queryState,jdbcType=INTEGER}, #{chatId,jdbcType=BIGINT}, #{score,jdbcType=INTEGER}, - #{feedback,jdbcType=VARCHAR}, #{queryText,jdbcType=LONGVARCHAR}, #{queryResult,jdbcType=LONGVARCHAR} + #{feedback,jdbcType=VARCHAR}, #{queryText,jdbcType=LONGVARCHAR}, #{queryResult,jdbcType=LONGVARCHAR}, + #{similarQueries, jdbcType=LONGVARCHAR} ) @@ -110,6 +117,9 @@ query_result = #{queryResult,jdbcType=LONGVARCHAR}, + + similar_queries = #{similarQueries,jdbcType=LONGVARCHAR}, + where question_id = #{questionId,jdbcType=BIGINT} diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 63cffdc14..c36956023 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -27,7 +27,8 @@ com.tencent.supersonic.chat.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.processor.parse.EntityInfoProcessor, \ com.tencent.supersonic.chat.processor.parse.SqlInfoProcessor, \ com.tencent.supersonic.chat.processor.parse.TimeCostProcessor, \ - com.tencent.supersonic.chat.processor.parse.RespBuildProcessor + com.tencent.supersonic.chat.processor.parse.RespBuildProcessor, \ + com.tencent.supersonic.chat.processor.parse.SimilarQueryProcessor com.tencent.supersonic.chat.api.component.SemanticInterpreter=\ com.tencent.supersonic.knowledge.semantic.LocalSemanticInterpreter @@ -42,7 +43,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor com.tencent.supersonic.chat.processor.execute.ExecuteResultProcessor=\ - com.tencent.supersonic.chat.processor.execute.SimilarMetricProcessor + com.tencent.supersonic.chat.processor.execute.SimilarMetricProcessor,\ + com.tencent.supersonic.chat.processor.execute.DimensionRecommendProcessor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 533bb84bd..ec574f5cd 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -37,6 +37,7 @@ CREATE TABLE `s2_chat_query` `query_result` mediumtext NOT NULL , `score` int DEFAULT '0', `feedback` varchar(1024) DEFAULT '', + `similar_queries` varchar(1024) DEFAULT '', PRIMARY KEY (`question_id`) ); diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 16b42cfe0..9543b5274 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -157,19 +157,21 @@ CREATE TABLE `s2_chat_parse` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE `s2_chat_query` ( - `question_id` bigint(20) NOT NULL AUTO_INCREMENT, - `agent_id` int(11) DEFAULT NULL, - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `query_text` mediumtext, - `user_name` varchar(150) DEFAULT NULL, - `query_state` int(1) DEFAULT NULL, - `chat_id` bigint(20) NOT NULL, - `query_result` mediumtext, - `score` int(11) DEFAULT '0', - `feedback` varchar(1024) DEFAULT '', - PRIMARY KEY (`question_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `s2_chat_query` +( + `question_id` bigint(20) NOT NULL AUTO_INCREMENT, + `agent_id` int(11) DEFAULT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `query_text` mediumtext, + `user_name` varchar(150) DEFAULT NULL, + `query_state` int(1) DEFAULT NULL, + `chat_id` bigint(20) NOT NULL, + `query_result` mediumtext, + `score` int(11) DEFAULT '0', + `feedback` varchar(1024) DEFAULT '', + `similar_queries` varchar(1024) DEFAULT '', + PRIMARY KEY (`question_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; CREATE TABLE `s2_chat_statistics` ( diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 94d20ad97..1ea1367be 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -142,4 +142,7 @@ CREATE TABLE `s2_metric_query_default_config` `updated_at` datetime null, `created_by` varchar(100) null, `updated_by` varchar(100) null -)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; \ No newline at end of file +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +--20231214 +alter table s2_chat_query add column `similar_queries` varchar(1024) DEFAULT ''; \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 954d50137..4fe039e90 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -37,6 +37,7 @@ CREATE TABLE `s2_chat_query` `query_result` mediumtext NOT NULL , `score` int DEFAULT '0', `feedback` varchar(1024) DEFAULT '', + `similar_queries` varchar(1024) DEFAULT '', PRIMARY KEY (`question_id`) ); @@ -200,6 +201,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` ( `alias` varchar(500) DEFAULT NULL, `tags` varchar(500) DEFAULT NULL, `relate_dimensions` varchar(500) DEFAULT NULL, + `ext` LONGVARCHAR DEFAULT NULL , PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_metric IS 'metric information table'; @@ -530,4 +532,16 @@ CREATE TABLE s2_sys_parameter id INT PRIMARY KEY AUTO_INCREMENT, admin varchar(500), parameters text null +); + +CREATE TABLE `s2_metric_query_default_config`( + `id` bigint NOT NULL AUTO_INCREMENT, + `metric_id` bigint, + `user_name` varchar(255) NOT NULL, + `default_config` varchar(1000) NOT NULL, + `created_at` TIMESTAMP null, + `updated_at` TIMESTAMP null, + `created_by` varchar(100) null, + `updated_by` varchar(100) not null, + PRIMARY KEY (`id`) ); \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java index 3be771bef..278d52095 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.api.model.response; +import com.google.common.base.Objects; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; @@ -62,4 +63,24 @@ public class ModelResp extends SchemaItem { return null; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + ModelResp that = (ModelResp) o; + return Objects.equal(getId(), that.getId()); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), getId()); + } + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index d1669f732..cca99cff6 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -295,16 +295,19 @@ public class MetricServiceImpl implements MetricService { } @Override - public void saveOrUpdateMetricQueryDefaultConfig(MetricQueryDefaultConfig queryDefaultConfig, User user) { - MetricQueryDefaultConfigDO metricQueryDefaultConfigDO = new MetricQueryDefaultConfigDO(); - if (queryDefaultConfig.getId() == null) { - queryDefaultConfig.createdBy(user.getName()); - BeanMapper.mapper(queryDefaultConfig, metricQueryDefaultConfigDO); - metricRepository.saveDefaultQueryConfig(metricQueryDefaultConfigDO); + public void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user) { + MetricQueryDefaultConfigDO defaultConfigDO = + metricRepository.getDefaultQueryConfig(defaultConfig.getMetricId(), user.getName()); + if (defaultConfigDO == null) { + defaultConfigDO = new MetricQueryDefaultConfigDO(); + defaultConfig.createdBy(user.getName()); + BeanMapper.mapper(defaultConfig, defaultConfigDO); + metricRepository.saveDefaultQueryConfig(defaultConfigDO); } else { - queryDefaultConfig.updatedBy(user.getName()); - BeanMapper.mapper(queryDefaultConfig, metricQueryDefaultConfigDO); - metricRepository.updateDefaultQueryConfig(metricQueryDefaultConfigDO); + defaultConfig.setId(defaultConfigDO.getId()); + defaultConfig.updatedBy(user.getName()); + BeanMapper.mapper(defaultConfig, defaultConfigDO); + metricRepository.updateDefaultQueryConfig(defaultConfigDO); } } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index a1406c54e..95dfa96de 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -15,7 +15,6 @@ import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; -import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.request.DateInfoReq; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; @@ -46,9 +45,7 @@ import com.tencent.supersonic.semantic.model.domain.manager.DatasourceYamlManage import com.tencent.supersonic.semantic.model.domain.manager.DimensionYamlManager; import com.tencent.supersonic.semantic.model.domain.manager.MetricYamlManager; import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; -import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; -import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; @@ -66,7 +63,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import java.util.ArrayList; -import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -307,8 +303,7 @@ public class ModelServiceImpl implements ModelService { modelRespSet = modelRespSet.stream().filter(modelResp -> modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet()); } - return fillMetricInfo(new ArrayList<>(modelRespSet)).stream() - .sorted(Comparator.comparingLong(SchemaItem::getId)).collect(Collectors.toList()); + return new ArrayList<>(modelRespSet); } public List getModelRespAuthInheritDomain(User user, AuthType authType) { @@ -464,21 +459,6 @@ public class ModelServiceImpl implements ModelService { modelRepository.batchUpdate(modelDOS); } - private List fillMetricInfo(List modelResps) { - if (CollectionUtils.isEmpty(modelResps)) { - return modelResps; - } - Map> metricMap = metricService.getMetrics(new MetricFilter()).stream() - .collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionMap = dimensionService.getDimensions(new DimensionFilter()).stream() - .collect(Collectors.groupingBy(DimensionResp::getModelId)); - modelResps.forEach(modelResp -> { - modelResp.setDimensionCnt(dimensionMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); - modelResp.setMetricCnt(metricMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); - }); - return modelResps; - } - protected ModelDO getModelDO(Long id) { return modelRepository.getModelById(id); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java index 2cb23b778..cc109cfaa 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.application; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ViewInfoReq; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; @@ -13,7 +14,6 @@ import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; -import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -44,11 +44,9 @@ public class ViewInfoServiceImpl { return viewInfoRepository.getViewInfoList(domainId); } - public List getDomainSchema(Long domainId) { + public List getDomainSchema(Long domainId, User user) { List domainSchemaRelaResps = Lists.newArrayList(); - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setDomainIds(Lists.newArrayList(domainId)); - List modelResps = modelService.getModelList(modelFilter); + List modelResps = modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); for (ModelResp modelResp : modelResps) { ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp(); MetaFilter metaFilter = new MetaFilter(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java index ca9149d38..e58e37d06 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java @@ -42,7 +42,7 @@ public interface MetricService { List getDataItems(Long modelId); - void saveOrUpdateMetricQueryDefaultConfig(MetricQueryDefaultConfig queryDefaultConfig, User user); + void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user); MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java index a2aed466d..aeb2d1e15 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java @@ -134,7 +134,7 @@ public class MetricController { HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - metricService.saveOrUpdateMetricQueryDefaultConfig(queryDefaultConfig, user); + metricService.saveMetricQueryDefaultConfig(queryDefaultConfig, user); return true; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java index a961547bf..97089978b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java @@ -47,8 +47,11 @@ public class ViewInfoController { } @GetMapping("/getDomainSchemaRela/{domainId}") - public List getDomainSchema(@PathVariable("domainId") Long domainId) { - return viewInfoServiceImpl.getDomainSchema(domainId); + public List getDomainSchema(@PathVariable("domainId") Long domainId, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return viewInfoServiceImpl.getDomainSchema(domainId, user); } } diff --git a/semantic/model/src/main/resources/mapper/DomainDOMapper.xml b/semantic/model/src/main/resources/mapper/DomainDOMapper.xml deleted file mode 100644 index a0d65c3a4..000000000 --- a/semantic/model/src/main/resources/mapper/DomainDOMapper.xml +++ /dev/null @@ -1,251 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, name, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, - admin, admin_org, is_open, viewer, view_org - - - - - delete from s2_domain - where id = #{id,jdbcType=BIGINT} - - - insert into s2_domain (id, name, biz_name, - parent_id, status, created_at, - created_by, updated_at, updated_by, - admin, admin_org, is_open, - viewer, view_org) - values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR}, - #{parentId,jdbcType=BIGINT}, #{status,jdbcType=INTEGER}, #{createdAt,jdbcType=TIMESTAMP}, - #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, - #{admin,jdbcType=VARCHAR}, #{adminOrg,jdbcType=VARCHAR}, #{isOpen,jdbcType=INTEGER}, - #{viewer,jdbcType=VARCHAR}, #{viewOrg,jdbcType=VARCHAR}) - - - insert into s2_domain - - - id, - - - name, - - - biz_name, - - - parent_id, - - - status, - - - created_at, - - - created_by, - - - updated_at, - - - updated_by, - - - admin, - - - admin_org, - - - is_open, - - - viewer, - - - view_org, - - - - - #{id,jdbcType=BIGINT}, - - - #{name,jdbcType=VARCHAR}, - - - #{bizName,jdbcType=VARCHAR}, - - - #{parentId,jdbcType=BIGINT}, - - - #{status,jdbcType=INTEGER}, - - - #{createdAt,jdbcType=TIMESTAMP}, - - - #{createdBy,jdbcType=VARCHAR}, - - - #{updatedAt,jdbcType=TIMESTAMP}, - - - #{updatedBy,jdbcType=VARCHAR}, - - - #{admin,jdbcType=VARCHAR}, - - - #{adminOrg,jdbcType=VARCHAR}, - - - #{isOpen,jdbcType=INTEGER}, - - - #{viewer,jdbcType=VARCHAR}, - - - #{viewOrg,jdbcType=VARCHAR}, - - - - - - update s2_domain - - - name = #{name,jdbcType=VARCHAR}, - - - biz_name = #{bizName,jdbcType=VARCHAR}, - - - parent_id = #{parentId,jdbcType=BIGINT}, - - - status = #{status,jdbcType=INTEGER}, - - - created_at = #{createdAt,jdbcType=TIMESTAMP}, - - - created_by = #{createdBy,jdbcType=VARCHAR}, - - - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - - - updated_by = #{updatedBy,jdbcType=VARCHAR}, - - - admin = #{admin,jdbcType=VARCHAR}, - - - admin_org = #{adminOrg,jdbcType=VARCHAR}, - - - is_open = #{isOpen,jdbcType=INTEGER}, - - - viewer = #{viewer,jdbcType=VARCHAR}, - - - view_org = #{viewOrg,jdbcType=VARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update s2_domain - set name = #{name,jdbcType=VARCHAR}, - biz_name = #{bizName,jdbcType=VARCHAR}, - parent_id = #{parentId,jdbcType=BIGINT}, - status = #{status,jdbcType=INTEGER}, - created_at = #{createdAt,jdbcType=TIMESTAMP}, - created_by = #{createdBy,jdbcType=VARCHAR}, - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - updated_by = #{updatedBy,jdbcType=VARCHAR}, - admin = #{admin,jdbcType=VARCHAR}, - admin_org = #{adminOrg,jdbcType=VARCHAR}, - is_open = #{isOpen,jdbcType=INTEGER}, - viewer = #{viewer,jdbcType=VARCHAR}, - view_org = #{viewOrg,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file