topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Chat) Format the results of the parse and query phases into text (#1045)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-05-29 17:02:45 +08:00
committed by GitHub
parent c7cb6df80b
commit 8aab359def
16 changed files with 119 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/DefaultShowType.java
View File

@@ -1,9 +0,0 @@
package com.tencent.supersonic.chat.api.pojo.enums;
public enum DefaultShowType {
TEXT,
TABLE,
WIDGET
}

1
chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java
View File

@@ -33,7 +33,6 @@ public class Agent extends RecordInfo {
private String agentConfig;
private LLMConfig llmConfig;
private MultiTurnConfig multiTurnConfig;
private VisualConfig visualConfig;
public List<String> getTools(AgentToolType type) {
Map map = JSONObject.parseObject(agentConfig, Map.class);

15
chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/VisualConfig.java
View File

@@ -1,15 +0,0 @@
package com.tencent.supersonic.chat.server.agent;
import com.tencent.supersonic.chat.api.pojo.enums.DefaultShowType;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class VisualConfig {
private DefaultShowType defaultShowType;
}

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java
View File

@@ -60,6 +60,4 @@ public class AgentDO {
private String multiTurnConfig;
private String visualConfig;
}

16
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.chat.server.processor.execute;
import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
import com.tencent.supersonic.chat.server.util.ResultFormatter;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
public class QueryResultFormatProcessor implements ExecuteResultProcessor {
@Override
public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) {
String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
queryResult.getQueryResults());
queryResult.setTextResult(textResult);
}
}

61
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java Normal file
View File

@@ -0,0 +1,61 @@
package com.tencent.supersonic.chat.server.processor.parse;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
public class ParseResultFormatProcessor implements ParseResultProcessor {
@Override
public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
formatParseResult(parseResp);
}
private void formatParseResult(ParseResp parseResp) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
for (SemanticParseInfo parseInfo : selectedParses) {
formatParseInfo(parseInfo);
}
}
private void formatParseInfo(SemanticParseInfo parseInfo) {
StringBuilder textBuilder = new StringBuilder();
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
metric.ifPresent(schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames = parseInfo.getDimensions().stream()
.map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionNames)) {
textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
}
textBuilder.append("\n**筛选条件:**\n");
if (parseInfo.getDateInfo() != null) {
textBuilder.append("数据时间:").append(parseInfo.getDateInfo().getStartDate()).append("~")
.append(parseInfo.getDateInfo().getEndDate()).append(" ");
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
|| CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
queryFilters.addAll(parseInfo.getMetricFilters());
for (QueryFilter queryFilter : queryFilters) {
textBuilder.append(queryFilter.getName())
.append(" ")
.append(queryFilter.getOperator().getValue())
.append(" ")
.append(queryFilter.getValue())
.append(" ");
}
}
parseInfo.setTextInfo(textBuilder.toString());
}
}

5
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java
View File

@@ -3,13 +3,12 @@ package com.tencent.supersonic.chat.server.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.server.agent.Agent;
import com.tencent.supersonic.headless.api.pojo.LLMConfig;
import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
import com.tencent.supersonic.chat.server.agent.VisualConfig;
import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO;
import com.tencent.supersonic.chat.server.persistence.mapper.AgentDOMapper;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.LLMConfig;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.List;
@@ -66,7 +65,6 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO>
agent.setExamples(JsonUtil.toList(agentDO.getExamples(), String.class));
agent.setLlmConfig(JsonUtil.toObject(agentDO.getLlmConfig(), LLMConfig.class));
agent.setMultiTurnConfig(JsonUtil.toObject(agentDO.getMultiTurnConfig(), MultiTurnConfig.class));
agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class));
return agent;
}
@@ -77,7 +75,6 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO>
agentDO.setExamples(JsonUtil.toString(agent.getExamples()));
agentDO.setLlmConfig(JsonUtil.toString(agent.getLlmConfig()));
agentDO.setMultiTurnConfig(JsonUtil.toString(agent.getMultiTurnConfig()));
agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig()));
if (agentDO.getStatus() == null) {
agentDO.setStatus(1);
}

32
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java Normal file
View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.chat.server.util;
import com.tencent.supersonic.common.pojo.QueryColumn;
import java.util.List;
import java.util.Map;
public class ResultFormatter {
public static String transform2TextNew(List<QueryColumn> queryColumns, List<Map<String, Object>> queryResults) {
StringBuilder table = new StringBuilder();
for (QueryColumn column : queryColumns) {
String columnName = column.getName();
table.append("| ").append(columnName).append(" ");
}
table.append("|\n");
for (int i = 0; i < queryColumns.size(); i++) {
table.append("|:---:");
}
table.append("|\n");
for (Map<String, Object> row : queryResults) {
for (QueryColumn column : queryColumns) {
String columnKey = column.getNameEn();
Object value = row.get(columnKey);
table.append("| ").append(value != null ? value.toString() : "").append(" ");
}
table.append("|\n");
}
return table.toString();
}
}

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java
View File

@@ -39,6 +39,7 @@ public class SemanticParseInfo {
private SqlInfo sqlInfo = new SqlInfo();
private QueryType queryType = QueryType.ID;
private EntityInfo entityInfo;
private String textInfo;
private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
@Override

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java
View File

@@ -22,6 +22,7 @@ public class QueryResult {
private SemanticParseInfo chatContext;
private Object response;
private List<Map<String, Object>> queryResults;
private String textResult;
private Long queryTimeCost;
private EntityInfo entityInfo;
private List<SchemaElement> recommendedDimensions;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/PromptGenerator.java
View File

@@ -115,8 +115,8 @@ public class PromptGenerator {
String currentDataStr = "当前的日期是" + currentDate;
String linkingListStr = String.join("，", priorLinkingList);
String termStr = getTermStr(llmReq);
String questionAugmented = String.format("%s (补充信息:%s . %s . %s) (备注: %s)", llmReq.getQueryText(), linkingListStr,
currentDataStr, termStr, priorExts);
String questionAugmented = String.format("%s (补充信息:%s . %s . %s) (备注: %s)", llmReq.getQueryText(),
linkingListStr, currentDataStr, termStr, priorExts);
return Pair.of(dbSchema, questionAugmented);
}
@@ -125,7 +125,7 @@ public class PromptGenerator {
StringBuilder termsDesc = new StringBuilder();
if (!CollectionUtils.isEmpty(terms)) {
termsDesc.append("相关业务术语：");
for (int idx = 0 ; idx < terms.size() ; idx++) {
for (int idx = 0; idx < terms.size(); idx++) {
LLMReq.Term term = terms.get(idx);
String name = term.getName();
String description = term.getDescription();

6
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -60,12 +60,14 @@ com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor,\
com.tencent.supersonic.chat.server.processor.parse.ParseResultFormatProcessor
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor,\
com.tencent.supersonic.chat.server.processor.execute.QueryResultFormatProcessor
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore

1
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -312,6 +312,5 @@ CREATE TABLE IF NOT EXISTS `s2_term` (
--20240520
alter table s2_agent add column `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL;
alter table s2_agent add column `multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL;
alter table s2_agent add column `visual_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL;
alter table s2_model add column `ext` varchar(1000) DEFAULT NULL;

1
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -354,7 +354,6 @@ CREATE TABLE IF NOT EXISTS s2_agent
config varchar(2000) null,
llm_config varchar(2000) null,
multi_turn_config varchar(2000) null,
visual_config varchar(2000) null,
created_by varchar(100) null,
created_at TIMESTAMP null,
updated_by varchar(100) null,

1
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -74,7 +74,6 @@ CREATE TABLE `s2_agent` (
`config` varchar(6000) COLLATE utf8_unicode_ci DEFAULT NULL,
`llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
`multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
`visual_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
`created_by` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
`created_at` datetime DEFAULT NULL,
`updated_by` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,

1
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -354,7 +354,6 @@ CREATE TABLE IF NOT EXISTS s2_agent
config varchar(2000) null,
llm_config varchar(2000) null,
multi_turn_config varchar(2000) null,
visual_config varchar(2000) null,
created_by varchar(100) null,
created_at TIMESTAMP null,
updated_by varchar(100) null,