topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) chat history record save query sql (#253)

Browse Source 
* (improvement)(chat) history record save query sql

* (improvement)(semantic) update easyexcel version

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-10-18 15:06:29 +08:00
committed by GitHub
parent 36052cb4f2
commit 2e4954a4e8
13 changed files with 132 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.Date;
import java.util.List;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import lombok.Data;
@Data
@@ -13,4 +15,5 @@ public class QueryResp {
private String feedback;
private String queryText;
private QueryResult queryResult;
private List<SemanticParseInfo> parseInfos;
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatParseMapper.java
View File

@@ -12,6 +12,10 @@ public interface ChatParseMapper {
boolean batchSaveParseInfo(@Param("list") List<ChatParseDO> list);
boolean updateParseInfo(ChatParseDO chatParseDO);
ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
List<ChatParseDO> getParseInfoList(List<Long> questionIds);
}

6
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java
View File

@@ -21,18 +21,22 @@ public interface ChatQueryRepository {
void createChatQuery(QueryResult queryResult, ChatContext chatCtx);
void updateChatParseInfo(List<ChatParseDO> chatParseDOS);
ChatQueryDO getLastChatQuery(long chatId);
int updateChatQuery(ChatQueryDO chatQueryDO);
Long createChatParse(ParseResp parseResult, ChatContext chatCtx, QueryReq queryReq);
Boolean batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
List<ChatParseDO> batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses);
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
List<ChatParseDO> getParseInfoList(List<Long> questionIds);
Boolean deleteChatQuery(Long questionId);
}

20
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java
View File

@@ -130,7 +130,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return queryId;
}
public Boolean batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
@Override
public List<ChatParseDO> batchSaveParseInfo(ChatContext chatCtx, QueryReq queryReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses) {
@@ -139,8 +140,15 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
log.info("candidateParses size:{},selectedParses size:{}", candidateParses.size(), selectedParses.size());
getChatParseDO(chatCtx, queryReq, queryId, 0, 1, candidateParses, chatParseDOList);
getChatParseDO(chatCtx, queryReq, queryId, candidateParses.size(), 0, selectedParses, chatParseDOList);
Boolean save = chatParseMapper.batchSaveParseInfo(chatParseDOList);
return save;
chatParseMapper.batchSaveParseInfo(chatParseDOList);
return chatParseDOList;
}
@Override
public void updateChatParseInfo(List<ChatParseDO> chatParseDOS) {
for (ChatParseDO chatParseDO : chatParseDOS) {
chatParseMapper.updateParseInfo(chatParseDO);
}
}
public void getChatParseDO(ChatContext chatCtx, QueryReq queryReq, Long queryId, int base, int isCandidate,
@@ -180,10 +188,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatQueryDOMapper.updateByPrimaryKeyWithBLOBs(chatQueryDO);
}
@Override
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId) {
return chatParseMapper.getParseInfo(questionId, userName, parseId);
}
@Override
public List<ChatParseDO> getParseInfoList(List<Long> questionIds) {
return chatParseMapper.getParseInfoList(questionIds);
}
@Override
public Boolean deleteChatQuery(Long questionId) {
return chatQueryDOMapper.deleteByPrimaryKey(questionId);

4
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.SemanticService;
@@ -16,7 +17,8 @@ import org.springframework.util.CollectionUtils;
public class EntityInfoParseResponder implements ParseResponder {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
public void fillResponse(ParseResp parseResp, QueryContext queryContext,
List<ChatParseDO> chatParseDOS) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
if (CollectionUtils.isEmpty(selectedParses)) {
return;

27
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java
View File

@@ -5,19 +5,44 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
public class ExplainSqlParseResponder implements ParseResponder {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
public void fillResponse(ParseResp parseResp, QueryContext queryContext,
List<ChatParseDO> chatParseDOS) {
QueryReq queryReq = queryContext.getRequest();
addExplainSql(queryReq, parseResp.getSelectedParses());
addExplainSql(queryReq, parseResp.getCandidateParses());
if (!CollectionUtils.isEmpty(chatParseDOS)) {
Map<Integer, ChatParseDO> chatParseDOMap = chatParseDOS.stream()
.collect(Collectors.toMap(ChatParseDO::getParseId,
Function.identity(), (oldValue, newValue) -> newValue));
updateParseInfo(chatParseDOMap, parseResp.getSelectedParses());
updateParseInfo(chatParseDOMap, parseResp.getCandidateParses());
}
}
private void updateParseInfo(Map<Integer, ChatParseDO> chatParseDOMap, List<SemanticParseInfo> parseInfos) {
if (CollectionUtils.isEmpty(parseInfos)) {
return;
}
for (SemanticParseInfo parseInfo : parseInfos) {
ChatParseDO chatParseDO = chatParseDOMap.get(parseInfo.getId());
if (chatParseDO != null) {
chatParseDO.setParseInfo(JsonUtil.toString(parseInfo));
}
}
}
private void addExplainSql(QueryReq queryReq, List<SemanticParseInfo> semanticParseInfos) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ParseResponder.java
View File

@@ -2,9 +2,11 @@ package com.tencent.supersonic.chat.responder.parse;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import java.util.List;
public interface ParseResponder {
void fillResponse(ParseResp parseResp, QueryContext queryContext);
void fillResponse(ParseResp parseResp, QueryContext queryContext, List<ChatParseDO> chatParseDOS);
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java
View File

@@ -49,11 +49,13 @@ public interface ChatService {
void addQuery(QueryResult queryResult, ChatContext chatCtx);
void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
List<ChatParseDO> batchAddParse(ChatContext chatCtx, QueryReq queryReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses);
void updateChatParse(List<ChatParseDO> chatParseDOS);
ChatQueryDO getLastQuery(long chatId);
int updateQuery(ChatQueryDO chatQueryDO);

34
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -20,6 +20,7 @@ import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
import com.tencent.supersonic.chat.persistence.repository.ChatRepository;
import java.text.SimpleDateFormat;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -133,7 +134,30 @@ public class ChatServiceImpl implements ChatService {
@Override
public PageInfo<QueryResp> queryInfo(PageQueryInfoReq pageQueryInfoReq, long chatId) {
return chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId);
PageInfo<QueryResp> queryRespPageInfo = chatQueryRepository.getChatQuery(pageQueryInfoReq, chatId);
if (CollectionUtils.isEmpty(queryRespPageInfo.getList())) {
return queryRespPageInfo;
}
List<Long> queryIds = queryRespPageInfo.getList().stream()
.map(QueryResp::getQuestionId).collect(Collectors.toList());
List<ChatParseDO> chatParseDOs = chatQueryRepository.getParseInfoList(queryIds);
if (CollectionUtils.isEmpty(chatParseDOs)) {
return queryRespPageInfo;
}
Map<Long, List<ChatParseDO>> chatParseMap = chatParseDOs.stream()
.collect(Collectors.groupingBy(ChatParseDO::getQuestionId));
for (QueryResp queryResp : queryRespPageInfo.getList()) {
List<ChatParseDO> chatParseDOList = chatParseMap.get(queryResp.getQuestionId());
if (CollectionUtils.isEmpty(chatParseMap)) {
continue;
}
List<SemanticParseInfo> parseInfos = chatParseDOList.stream().map(chatParseDO ->
JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class))
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
.collect(Collectors.toList());
queryResp.setParseInfos(parseInfos);
}
return queryRespPageInfo;
}
@Override
@@ -173,12 +197,16 @@ public class ChatServiceImpl implements ChatService {
}
@Override
public void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
public List<ChatParseDO> batchAddParse(ChatContext chatCtx, QueryReq queryReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses) {
chatQueryRepository.batchSaveParseInfo(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
return chatQueryRepository.batchSaveParseInfo(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
}
@Override
public void updateChatParse(List<ChatParseDO> chatParseDOS) {
chatQueryRepository.updateChatParseInfo(chatParseDOS);
}
@Override

7
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -63,6 +63,7 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -113,6 +114,7 @@ public class QueryServiceImpl implements QueryService {
log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
ParseResp parseResult;
List<ChatParseDO> chatParseDOS = Lists.newArrayList();
if (queryCtx.getCandidateQueries().size() > 0) {
log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
Collectors.toList()));
@@ -130,7 +132,7 @@ public class QueryServiceImpl implements QueryService {
.selectedParses(selectedParses)
.candidateParses(candidateParses)
.build();
chatService.batchAddParse(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
chatParseDOS = chatService.batchAddParse(chatCtx, queryReq, parseResult, candidateParses, selectedParses);
saveInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(),
queryReq.getUser().getName(), queryReq.getChatId().longValue());
} else {
@@ -141,8 +143,9 @@ public class QueryServiceImpl implements QueryService {
.build();
}
for (ParseResponder parseResponder : parseResponders) {
parseResponder.fillResponse(parseResult, queryCtx);
parseResponder.fillResponse(parseResult, queryCtx, chatParseDOS);
}
chatService.updateChatParse(chatParseDOS);
return parseResult;
}

15
chat/core/src/main/resources/mapper/ChatParseMapper.xml
View File

@@ -25,6 +25,12 @@
</foreach>
</insert>
<update id="updateParseInfo" parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO">
update s2_chat_parse
set parse_info = #{parseInfo}
where parse_id = #{parseId}
</update>
<select id="getParseInfo" resultMap="ChatParse">
select *
from s2_chat_parse
@@ -32,4 +38,13 @@
and parse_id = #{parseId} limit 1
</select>
<select id="getParseInfoList" resultMap="ChatParse">
select *
from s2_chat_parse
where question_id in
<foreach item="questionId" index="index" collection="list" open="(" separator="," close=")">
#{questionId}
</foreach>
</select>
</mapper>

3
pom.xml
View File

@@ -70,7 +70,8 @@
<spotless.python.includes></spotless.python.includes>
<!-- Do not bump black version as decided by spotless maven plugin-->
<spotless.python.black.version>22.3.0</spotless.python.black.version>
<easyexcel.version>3.1.1</easyexcel.version>
<easyexcel.version>2.2.6</easyexcel.version>
<poi.version>3.17</poi.version>
</properties>
<dependencyManagement>

15
semantic/query/pom.xml
View File

@@ -97,6 +97,21 @@
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>${poi.version}</version>
</dependency>
</dependencies>