(improvement)(chat) chat history record save query sql (#253)

* (improvement)(chat) history record save query sql

* (improvement)(semantic) update easyexcel version

---------

Co-authored-by: jolunoluo

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java

@@ -1,6 +1,8 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
 import java.util.Date;
+import java.util.List;
+import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import lombok.Data;
 
 @Data
@@ -13,4 +15,5 @@ public class QueryResp {
     private String feedback;
     private String queryText;
     private QueryResult queryResult;
+    private List<SemanticParseInfo> parseInfos;
 }

chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatParseMapper.java

@@ -12,6 +12,10 @@ public interface ChatParseMapper {
 
     boolean batchSaveParseInfo(@Param("list") List<ChatParseDO> list);
 
+    boolean updateParseInfo(ChatParseDO chatParseDO);
+
     ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
 
+    List<ChatParseDO> getParseInfoList(List<Long> questionIds);
+
 }

chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java

@@ -21,18 +21,22 @@ public interface ChatQueryRepository {
 
     void createChatQuery(QueryResult queryResult, ChatContext chatCtx);
 
+    void updateChatParseInfo(List<ChatParseDO> chatParseDOS);
+
     ChatQueryDO getLastChatQuery(long chatId);
 
     int updateChatQuery(ChatQueryDO chatQueryDO);
 
     Long createChatParse(ParseResp parseResult, ChatContext chatCtx, QueryReq queryReq);
 
-    Boolean batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
+    List<ChatParseDO> batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
                                ParseResp parseResult,
                                List<SemanticParseInfo> candidateParses,
                                List<SemanticParseInfo> selectedParses);
 
     public ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
 
+    List<ChatParseDO> getParseInfoList(List<Long> questionIds);
+
     Boolean deleteChatQuery(Long questionId);
 }

chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java

@@ -130,7 +130,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
         return queryId;
     }
 
-    public Boolean batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
+    @Override
+    public List<ChatParseDO> batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
                                       ParseResp parseResult,
                                       List<SemanticParseInfo> candidateParses,
                                       List<SemanticParseInfo> selectedParses) {
@@ -139,8 +140,15 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
         log.info("candidateParses size:{},selectedParses size:{}", candidateParses.size(), selectedParses.size());
         getChatParseDO(chatCtx, queryReq, queryId, 0, 1, candidateParses, chatParseDOList);
         getChatParseDO(chatCtx, queryReq, queryId, candidateParses.size(), 0, selectedParses, chatParseDOList);
-        Boolean save = chatParseMapper.batchSaveParseInfo(chatParseDOList);
+        chatParseMapper.batchSaveParseInfo(chatParseDOList);
-        return save;
+        return chatParseDOList;
+    }
+
+    @Override
+    public void updateChatParseInfo(List<ChatParseDO> chatParseDOS) {
+        for (ChatParseDO chatParseDO : chatParseDOS) {
+            chatParseMapper.updateParseInfo(chatParseDO);
+        }
     }
 
     public void getChatParseDO(ChatContext chatCtx, QueryReq queryReq, Long queryId, int base, int isCandidate,
@@ -180,10 +188,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
         return chatQueryDOMapper.updateByPrimaryKeyWithBLOBs(chatQueryDO);
     }
 
+    @Override
     public ChatParseDO getParseInfo(Long questionId, String userName, int parseId) {
         return chatParseMapper.getParseInfo(questionId, userName, parseId);
     }
 
+    @Override
+    public List<ChatParseDO> getParseInfoList(List<Long> questionIds) {
+        return chatParseMapper.getParseInfoList(questionIds);
+    }
+
     @Override
     public Boolean deleteChatQuery(Long questionId) {
         return chatQueryDOMapper.deleteByPrimaryKey(questionId);

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java

@@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
+import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.query.QueryManager;
 import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
 import com.tencent.supersonic.chat.service.SemanticService;
@@ -16,7 +17,8 @@ import org.springframework.util.CollectionUtils;
 public class EntityInfoParseResponder implements ParseResponder {
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
+    public void fillResponse(ParseResp parseResp, QueryContext queryContext,
+                             List<ChatParseDO> chatParseDOS) {
         List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
         if (CollectionUtils.isEmpty(selectedParses)) {
             return;

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java

@@ -5,19 +5,44 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
+import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.query.QueryManager;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import org.springframework.util.CollectionUtils;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class ExplainSqlParseResponder implements ParseResponder {
 
     @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
+    public void fillResponse(ParseResp parseResp, QueryContext queryContext,
+                             List<ChatParseDO> chatParseDOS) {
         QueryReq queryReq = queryContext.getRequest();
         addExplainSql(queryReq, parseResp.getSelectedParses());
         addExplainSql(queryReq, parseResp.getCandidateParses());
+        if (!CollectionUtils.isEmpty(chatParseDOS)) {
+            Map<Integer, ChatParseDO> chatParseDOMap = chatParseDOS.stream()
+                    .collect(Collectors.toMap(ChatParseDO::getParseId,
+                            Function.identity(), (oldValue, newValue) -> newValue));
+            updateParseInfo(chatParseDOMap, parseResp.getSelectedParses());
+            updateParseInfo(chatParseDOMap, parseResp.getCandidateParses());
+        }
+    }
+
+    private void updateParseInfo(Map<Integer, ChatParseDO> chatParseDOMap, List<SemanticParseInfo> parseInfos) {
+        if (CollectionUtils.isEmpty(parseInfos)) {
+            return;
+        }
+        for (SemanticParseInfo parseInfo : parseInfos) {
+            ChatParseDO chatParseDO = chatParseDOMap.get(parseInfo.getId());
+            if (chatParseDO != null) {
+                chatParseDO.setParseInfo(JsonUtil.toString(parseInfo));
+            }
+        }
     }
 
     private void addExplainSql(QueryReq queryReq, List<SemanticParseInfo> semanticParseInfos) {

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ParseResponder.java

@@ -2,9 +2,11 @@ package com.tencent.supersonic.chat.responder.parse;
 
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
+import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
+import java.util.List;
 
 public interface ParseResponder {
 
-    void fillResponse(ParseResp parseResp, QueryContext queryContext);
+    void fillResponse(ParseResp parseResp, QueryContext queryContext, List<ChatParseDO> chatParseDOS);
 
 }

chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java

@@ -49,11 +49,13 @@ public interface ChatService {
 
     void addQuery(QueryResult queryResult, ChatContext chatCtx);
 
-    void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
+    List<ChatParseDO> batchAddParse(ChatContext chatCtx, QueryReq queryReq,
                          ParseResp parseResult,
                          List<SemanticParseInfo> candidateParses,
                          List<SemanticParseInfo> selectedParses);
 
+    void updateChatParse(List<ChatParseDO> chatParseDOS);
+
     ChatQueryDO getLastQuery(long chatId);
 
     int updateQuery(ChatQueryDO chatQueryDO);

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java

@@ -20,6 +20,7 @@ import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
 import com.tencent.supersonic.chat.persistence.repository.ChatRepository;
 
 import java.text.SimpleDateFormat;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -133,7 +134,30 @@ public class ChatServiceImpl implements ChatService {
 
     @Override
     public PageInfo<QueryResp> queryInfo(PageQueryInfoReq pageQueryInfoReq, long chatId) {
-        return chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId);
+        PageInfo<QueryResp> queryRespPageInfo = chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId);
+        if (CollectionUtils.isEmpty(queryRespPageInfo.getList())) {
+            return queryRespPageInfo;
+        }
+        List<Long> queryIds = queryRespPageInfo.getList().stream()
+                .map(QueryResp::getQuestionId).collect(Collectors.toList());
+        List<ChatParseDO> chatParseDOs = chatQueryRepository.getParseInfoList(queryIds);
+        if (CollectionUtils.isEmpty(chatParseDOs)) {
+            return queryRespPageInfo;
+        }
+        Map<Long, List<ChatParseDO>> chatParseMap = chatParseDOs.stream()
+                .collect(Collectors.groupingBy(ChatParseDO::getQuestionId));
+        for (QueryResp queryResp : queryRespPageInfo.getList()) {
+            List<ChatParseDO> chatParseDOList = chatParseMap.get(queryResp.getQuestionId());
+            if (CollectionUtils.isEmpty(chatParseMap)) {
+                continue;
+            }
+            List<SemanticParseInfo> parseInfos = chatParseDOList.stream().map(chatParseDO ->
+                    JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class))
+                    .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
+                    .collect(Collectors.toList());
+            queryResp.setParseInfos(parseInfos);
+        }
+        return queryRespPageInfo;
     }
 
     @Override
@@ -173,12 +197,16 @@ public class ChatServiceImpl implements ChatService {
     }
 
     @Override
-    public void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
+    public List<ChatParseDO> batchAddParse(ChatContext chatCtx, QueryReq queryReq,
             ParseResp parseResult,
             List<SemanticParseInfo> candidateParses,
             List<SemanticParseInfo> selectedParses) {
-        chatQueryRepository.batchSaveParseInfo(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
+        return chatQueryRepository.batchSaveParseInfo(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
+    }
 
+    @Override
+    public void updateChatParse(List<ChatParseDO> chatParseDOS) {
+        chatQueryRepository.updateChatParseInfo(chatParseDOS);
     }
 
     @Override

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -63,6 +63,7 @@ import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.compress.utils.Lists;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -113,6 +114,7 @@ public class QueryServiceImpl implements QueryService {
             log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
         });
         ParseResp parseResult;
+        List<ChatParseDO> chatParseDOS = Lists.newArrayList();
         if (queryCtx.getCandidateQueries().size() > 0) {
             log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
                     Collectors.toList()));
@@ -130,7 +132,7 @@ public class QueryServiceImpl implements QueryService {
                     .selectedParses(selectedParses)
                     .candidateParses(candidateParses)
                     .build();
-            chatService.batchAddParse(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
+            chatParseDOS = chatService.batchAddParse(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
             saveInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(),
                     queryReq.getUser().getName(), queryReq.getChatId().longValue());
         } else {
@@ -141,8 +143,9 @@ public class QueryServiceImpl implements QueryService {
                     .build();
         }
         for (ParseResponder parseResponder : parseResponders) {
-            parseResponder.fillResponse(parseResult, queryCtx);
+            parseResponder.fillResponse(parseResult, queryCtx, chatParseDOS);
         }
+        chatService.updateChatParse(chatParseDOS);
         return parseResult;
     }
 

chat/core/src/main/resources/mapper/ChatParseMapper.xml

@@ -25,6 +25,12 @@
         </foreach>
     </insert>
 
+    <update id="updateParseInfo" parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO">
+        update s2_chat_parse
+        set parse_info = #{parseInfo}
+        where parse_id = #{parseId}
+    </update>
+
     <select id="getParseInfo" resultMap="ChatParse">
         select *
         from s2_chat_parse
@@ -32,4 +38,13 @@
         and parse_id = #{parseId} limit 1
     </select>
 
+    <select id="getParseInfoList" resultMap="ChatParse">
+        select *
+        from s2_chat_parse
+        where question_id in
+        <foreach item="questionId" index="index" collection="list" open="(" separator="," close=")">
+            #{questionId}
+        </foreach>
+    </select>
+
 </mapper>

pom.xml

@@ -70,7 +70,8 @@
         <spotless.python.includes></spotless.python.includes>
         <!-- Do not bump black version as decided by spotless maven plugin-->
         <spotless.python.black.version>22.3.0</spotless.python.black.version>
-        <easyexcel.version>3.1.1</easyexcel.version>
+        <easyexcel.version>2.2.6</easyexcel.version>
+        <poi.version>3.17</poi.version>
     </properties>
 
     <dependencyManagement>

semantic/query/pom.xml

@@ -97,6 +97,21 @@
             <artifactId>easyexcel</artifactId>
             <version>${easyexcel.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>${poi.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>${poi.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml-schemas</artifactId>
+            <version>${poi.version}</version>
+        </dependency>
     </dependencies>