From 8aab359def8b0c48bdd42f118a96c9325f212bbf Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Wed, 29 May 2024 17:02:45 +0800 Subject: [PATCH] (improvement)(Chat) Format the results of the parse and query phases into text (#1045) Co-authored-by: jolunoluo --- .../chat/api/pojo/enums/DefaultShowType.java | 9 --- .../supersonic/chat/server/agent/Agent.java | 1 - .../chat/server/agent/VisualConfig.java | 15 ----- .../persistence/dataobject/AgentDO.java | 2 - .../execute/QueryResultFormatProcessor.java | 16 +++++ .../parse/ParseResultFormatProcessor.java | 61 +++++++++++++++++++ .../server/service/impl/AgentServiceImpl.java | 5 +- .../chat/server/util/ResultFormatter.java | 32 ++++++++++ .../headless/api/pojo/SemanticParseInfo.java | 1 + .../api/pojo/response/QueryResult.java | 1 + .../core/chat/parser/llm/PromptGenerator.java | 6 +- .../main/resources/META-INF/spring.factories | 6 +- .../resources/config.update/sql-update.sql | 1 - .../src/main/resources/db/schema-h2.sql | 1 - .../src/main/resources/db/schema-mysql.sql | 1 - .../src/test/resources/db/schema-h2.sql | 1 - 16 files changed, 119 insertions(+), 40 deletions(-) delete mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/DefaultShowType.java delete mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/VisualConfig.java create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/DefaultShowType.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/DefaultShowType.java deleted file mode 100644 index 08829a417..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/enums/DefaultShowType.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo.enums; - -public enum DefaultShowType { - - TEXT, - TABLE, - WIDGET - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java index 1224947d3..1eb15676d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java @@ -33,7 +33,6 @@ public class Agent extends RecordInfo { private String agentConfig; private LLMConfig llmConfig; private MultiTurnConfig multiTurnConfig; - private VisualConfig visualConfig; public List getTools(AgentToolType type) { Map map = JSONObject.parseObject(agentConfig, Map.class); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/VisualConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/VisualConfig.java deleted file mode 100644 index f6ee1fc78..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/VisualConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.tencent.supersonic.chat.server.agent; - -import com.tencent.supersonic.chat.api.pojo.enums.DefaultShowType; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class VisualConfig { - - private DefaultShowType defaultShowType; - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java index 42c1a075a..e84ae220d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java @@ -60,6 +60,4 @@ public class AgentDO { private String multiTurnConfig; - private String visualConfig; - } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java new file mode 100644 index 000000000..b493b52b5 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/QueryResultFormatProcessor.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.chat.server.processor.execute; + +import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext; +import com.tencent.supersonic.chat.server.util.ResultFormatter; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; + +public class QueryResultFormatProcessor implements ExecuteResultProcessor { + + @Override + public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) { + String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(), + queryResult.getQueryResults()); + queryResult.setTextResult(textResult); + } + +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java new file mode 100644 index 000000000..7d39dc9d6 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultFormatProcessor.java @@ -0,0 +1,61 @@ +package com.tencent.supersonic.chat.server.processor.parse; + +import com.tencent.supersonic.chat.server.pojo.ChatParseContext; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class ParseResultFormatProcessor implements ParseResultProcessor { + + @Override + public void process(ChatParseContext chatParseContext, ParseResp parseResp) { + formatParseResult(parseResp); + } + + private void formatParseResult(ParseResp parseResp) { + List selectedParses = parseResp.getSelectedParses(); + for (SemanticParseInfo parseInfo : selectedParses) { + formatParseInfo(parseInfo); + } + } + + private void formatParseInfo(SemanticParseInfo parseInfo) { + StringBuilder textBuilder = new StringBuilder(); + textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" "); + Optional metric = parseInfo.getMetrics().stream().findFirst(); + metric.ifPresent(schemaElement -> + textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" ")); + List dimensionNames = parseInfo.getDimensions().stream() + .map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(dimensionNames)) { + textBuilder.append("**维度:** ").append(String.join(",", dimensionNames)); + } + textBuilder.append("\n**筛选条件:**\n"); + if (parseInfo.getDateInfo() != null) { + textBuilder.append("数据时间:").append(parseInfo.getDateInfo().getStartDate()).append("~") + .append(parseInfo.getDateInfo().getEndDate()).append(" "); + } + if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters()) + || CollectionUtils.isEmpty(parseInfo.getMetricFilters())) { + Set queryFilters = parseInfo.getDimensionFilters(); + queryFilters.addAll(parseInfo.getMetricFilters()); + for (QueryFilter queryFilter : queryFilters) { + textBuilder.append(queryFilter.getName()) + .append(" ") + .append(queryFilter.getOperator().getValue()) + .append(" ") + .append(queryFilter.getValue()) + .append(" "); + } + } + parseInfo.setTextInfo(textBuilder.toString()); + } + +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index f514e67c3..d7cd44790 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -3,13 +3,12 @@ package com.tencent.supersonic.chat.server.service.impl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.server.agent.Agent; -import com.tencent.supersonic.headless.api.pojo.LLMConfig; import com.tencent.supersonic.chat.server.agent.MultiTurnConfig; -import com.tencent.supersonic.chat.server.agent.VisualConfig; import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO; import com.tencent.supersonic.chat.server.persistence.mapper.AgentDOMapper; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.LLMConfig; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import java.util.List; @@ -66,7 +65,6 @@ public class AgentServiceImpl extends ServiceImpl agent.setExamples(JsonUtil.toList(agentDO.getExamples(), String.class)); agent.setLlmConfig(JsonUtil.toObject(agentDO.getLlmConfig(), LLMConfig.class)); agent.setMultiTurnConfig(JsonUtil.toObject(agentDO.getMultiTurnConfig(), MultiTurnConfig.class)); - agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class)); return agent; } @@ -77,7 +75,6 @@ public class AgentServiceImpl extends ServiceImpl agentDO.setExamples(JsonUtil.toString(agent.getExamples())); agentDO.setLlmConfig(JsonUtil.toString(agent.getLlmConfig())); agentDO.setMultiTurnConfig(JsonUtil.toString(agent.getMultiTurnConfig())); - agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig())); if (agentDO.getStatus() == null) { agentDO.setStatus(1); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java new file mode 100644 index 000000000..6c9d57967 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java @@ -0,0 +1,32 @@ +package com.tencent.supersonic.chat.server.util; + +import com.tencent.supersonic.common.pojo.QueryColumn; + +import java.util.List; +import java.util.Map; + +public class ResultFormatter { + + public static String transform2TextNew(List queryColumns, List> queryResults) { + StringBuilder table = new StringBuilder(); + for (QueryColumn column : queryColumns) { + String columnName = column.getName(); + table.append("| ").append(columnName).append(" "); + } + table.append("|\n"); + for (int i = 0; i < queryColumns.size(); i++) { + table.append("|:---:"); + } + table.append("|\n"); + for (Map row : queryResults) { + for (QueryColumn column : queryColumns) { + String columnKey = column.getNameEn(); + Object value = row.get(columnKey); + table.append("| ").append(value != null ? value.toString() : "").append(" "); + } + table.append("|\n"); + } + return table.toString(); + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index f95278d24..0707981f0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -39,6 +39,7 @@ public class SemanticParseInfo { private SqlInfo sqlInfo = new SqlInfo(); private QueryType queryType = QueryType.ID; private EntityInfo entityInfo; + private String textInfo; private static class SchemaNameLengthComparator implements Comparator { @Override diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java index 75cab62a6..06410978b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java @@ -22,6 +22,7 @@ public class QueryResult { private SemanticParseInfo chatContext; private Object response; private List> queryResults; + private String textResult; private Long queryTimeCost; private EntityInfo entityInfo; private List recommendedDimensions; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/PromptGenerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/PromptGenerator.java index afbc5a28d..c5381d7bf 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/PromptGenerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/PromptGenerator.java @@ -115,8 +115,8 @@ public class PromptGenerator { String currentDataStr = "当前的日期是" + currentDate; String linkingListStr = String.join("，", priorLinkingList); String termStr = getTermStr(llmReq); - String questionAugmented = String.format("%s (补充信息:%s . %s . %s) (备注: %s)", llmReq.getQueryText(), linkingListStr, - currentDataStr, termStr, priorExts); + String questionAugmented = String.format("%s (补充信息:%s . %s . %s) (备注: %s)", llmReq.getQueryText(), + linkingListStr, currentDataStr, termStr, priorExts); return Pair.of(dbSchema, questionAugmented); } @@ -125,7 +125,7 @@ public class PromptGenerator { StringBuilder termsDesc = new StringBuilder(); if (!CollectionUtils.isEmpty(terms)) { termsDesc.append("相关业务术语："); - for (int idx = 0 ; idx < terms.size() ; idx++) { + for (int idx = 0; idx < terms.size(); idx++) { LLMReq.Term term = terms.get(idx); String name = term.getName(); String description = term.getDescription(); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index e0750b1e4..8a05fb484 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -60,12 +60,14 @@ com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\ com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\ - com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor + com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor,\ + com.tencent.supersonic.chat.server.processor.parse.ParseResultFormatProcessor com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\ com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\ com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\ - com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor + com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor,\ + com.tencent.supersonic.chat.server.processor.execute.QueryResultFormatProcessor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index e347ab880..05b7c3c66 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -312,6 +312,5 @@ CREATE TABLE IF NOT EXISTS `s2_term` ( --20240520 alter table s2_agent add column `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; alter table s2_agent add column `multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; -alter table s2_agent add column `visual_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; alter table s2_model add column `ext` varchar(1000) DEFAULT NULL; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index b3df0db95..8981ebcfb 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -354,7 +354,6 @@ CREATE TABLE IF NOT EXISTS s2_agent config varchar(2000) null, llm_config varchar(2000) null, multi_turn_config varchar(2000) null, - visual_config varchar(2000) null, created_by varchar(100) null, created_at TIMESTAMP null, updated_by varchar(100) null, diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 3e5dc58ab..e2eb69d46 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -74,7 +74,6 @@ CREATE TABLE `s2_agent` ( `config` varchar(6000) COLLATE utf8_unicode_ci DEFAULT NULL, `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL, `multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL, - `visual_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL, `created_by` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, `created_at` datetime DEFAULT NULL, `updated_by` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 36289fd32..ac7a9549b 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -354,7 +354,6 @@ CREATE TABLE IF NOT EXISTS s2_agent config varchar(2000) null, llm_config varchar(2000) null, multi_turn_config varchar(2000) null, - visual_config varchar(2000) null, created_by varchar(100) null, created_at TIMESTAMP null, updated_by varchar(100) null,