(improvement)(launcher) Optimize the demo to prevent the dimension value dictionary from being loaded after the demo is loaded. (#1059)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-05-30 22:26:20 +08:00
committed by GitHub
parent 4e6c076481
commit c000dfeb06
14 changed files with 161 additions and 182 deletions

View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.server.executor;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
import com.tencent.supersonic.chat.server.util.ResultFormatter;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
@@ -20,7 +21,11 @@ public class SqlExecutor implements ChatExecutor {
}
ExecuteQueryReq executeQueryReq = buildExecuteReq(chatExecuteContext);
ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
return chatQueryService.performExecution(executeQueryReq);
QueryResult queryResult = chatQueryService.performExecution(executeQueryReq);
String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
queryResult.getQueryResults());
queryResult.setTextResult(textResult);
return queryResult;
}
private ExecuteQueryReq buildExecuteReq(ChatExecuteContext chatExecuteContext) {

View File

@@ -1,16 +1,22 @@
package com.tencent.supersonic.chat.server.parser;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
public class NL2SQLParser implements ChatParser {
@@ -31,6 +37,7 @@ public class NL2SQLParser implements ChatParser {
parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
}
parseResp.getParseTimeCost().setSqlTime(text2SqlParseResp.getParseTimeCost().getSqlTime());
formatParseResult(parseResp);
}
private boolean checkSkip(ParseResp parseResp) {
@@ -43,4 +50,49 @@ public class NL2SQLParser implements ChatParser {
return false;
}
private void formatParseResult(ParseResp parseResp) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
for (SemanticParseInfo parseInfo : selectedParses) {
formatParseInfo(parseInfo);
}
}
private void formatParseInfo(SemanticParseInfo parseInfo) {
if (!PluginQueryManager.isPluginQuery(parseInfo.getQueryMode())) {
formatNL2SQLParseInfo(parseInfo);
}
}
private void formatNL2SQLParseInfo(SemanticParseInfo parseInfo) {
StringBuilder textBuilder = new StringBuilder();
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
metric.ifPresent(schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames = parseInfo.getDimensions().stream()
.map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionNames)) {
textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
}
textBuilder.append("\n\n**筛选条件:** \n");
if (parseInfo.getDateInfo() != null) {
textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~")
.append(parseInfo.getDateInfo().getEndDate()).append(" ");
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
|| CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
queryFilters.addAll(parseInfo.getMetricFilters());
for (QueryFilter queryFilter : queryFilters) {
textBuilder.append("**").append(queryFilter.getName()).append("**")
.append(" ")
.append(queryFilter.getOperator().getValue())
.append(" ")
.append(queryFilter.getValue())
.append(" ");
}
}
parseInfo.setTextInfo(textBuilder.toString());
}
}

View File

@@ -55,8 +55,13 @@ public class WebServiceQuery extends PluginSemanticQuery {
log.info("webServiceResponse result:{}", JsonUtil.toString(object));
try {
Map<String, Object> data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class);
if (data.get("resultList") != null) {
queryResult.setQueryResults((List<Map<String, Object>>) data.get("resultList"));
}
if (data.get("columns") != null) {
queryResult.setQueryColumns((List<QueryColumn>) data.get("columns"));
}
queryResult.setTextResult(String.valueOf(data.get("textInfo")));
queryResult.setQueryState(QueryState.SUCCESS);
} catch (Exception e) {
log.info("webServiceResponse result has an exception:{}", e.getMessage());

View File

@@ -86,6 +86,7 @@ public abstract class PluginRecognizer {
properties.put("name", plugin.getName());
semanticParseInfo.setProperties(properties);
semanticParseInfo.setScore(distance);
semanticParseInfo.setTextInfo(String.format("将由插件工具**%s**来解答", plugin.getName()));
fillSemanticParseInfo(semanticParseInfo);
return semanticParseInfo;
}

View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.chat.server.processor.execute;
import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
import com.tencent.supersonic.chat.server.util.ResultFormatter;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
public class QueryResultFormatProcessor implements ExecuteResultProcessor {
@Override
public void process(ChatExecuteContext chatExecuteContext, QueryResult queryResult) {
String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
queryResult.getQueryResults());
queryResult.setTextResult(textResult);
}
}

View File

@@ -1,61 +0,0 @@
package com.tencent.supersonic.chat.server.processor.parse;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
public class ParseResultFormatProcessor implements ParseResultProcessor {
@Override
public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
formatParseResult(parseResp);
}
private void formatParseResult(ParseResp parseResp) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
for (SemanticParseInfo parseInfo : selectedParses) {
formatParseInfo(parseInfo);
}
}
private void formatParseInfo(SemanticParseInfo parseInfo) {
StringBuilder textBuilder = new StringBuilder();
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
metric.ifPresent(schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames = parseInfo.getDimensions().stream()
.map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionNames)) {
textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
}
textBuilder.append("\n\n**筛选条件:** \n");
if (parseInfo.getDateInfo() != null) {
textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~")
.append(parseInfo.getDateInfo().getEndDate()).append(" ");
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
|| CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
queryFilters.addAll(parseInfo.getMetricFilters());
for (QueryFilter queryFilter : queryFilters) {
textBuilder.append("**").append(queryFilter.getName()).append("**")
.append(" ")
.append(queryFilter.getOperator().getValue())
.append(" ")
.append(queryFilter.getValue())
.append(" ");
}
}
parseInfo.setTextInfo(textBuilder.toString());
}
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.server.util;
import com.tencent.supersonic.common.pojo.QueryColumn;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
@@ -8,6 +9,9 @@ import java.util.Map;
public class ResultFormatter {
public static String transform2TextNew(List<QueryColumn> queryColumns, List<Map<String, Object>> queryResults) {
if (CollectionUtils.isEmpty(queryColumns)) {
return "";
}
StringBuilder table = new StringBuilder();
for (QueryColumn column : queryColumns) {
String columnName = column.getName();
@@ -18,6 +22,9 @@ public class ResultFormatter {
table.append("|:---:");
}
table.append("|\n");
if (queryResults == null) {
return table.toString();
}
for (Map<String, Object> row : queryResults) {
for (QueryColumn column : queryColumns) {
String columnKey = column.getNameEn();

View File

@@ -1,82 +0,0 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.server.service.impl.WordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Slf4j
@Component
@Order(2)
public class ApplicationStartedListener implements CommandLineRunner {
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private WordService wordService;
@Override
public void run(String... args) {
updateKnowledgeDimValue();
}
public Boolean updateKnowledgeDimValue() {
Boolean isOk = false;
try {
log.debug("ApplicationStartedInit start");
List<DictWord> dictWords = wordService.getAllDictWords();
wordService.setPreDictWords(dictWords);
knowledgeBaseService.reloadAllData(dictWords);
log.debug("ApplicationStartedInit end");
isOk = true;
} catch (Exception e) {
log.error("ApplicationStartedInit error", e);
}
return isOk;
}
public Boolean updateKnowledgeDimValueAsync() {
CompletableFuture.supplyAsync(() -> {
updateKnowledgeDimValue();
return null;
});
return true;
}
/***
* reload knowledge task
*/
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
public void reloadKnowledge() {
log.debug("reloadKnowledge start");
try {
List<DictWord> dictWords = wordService.getAllDictWords();
List<DictWord> preDictWords = wordService.getPreDictWords();
if (CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
log.debug("dictWords has not changed, reloadKnowledge end");
return;
}
log.info("dictWords has changed");
wordService.setPreDictWords(dictWords);
knowledgeBaseService.updateOnlineKnowledge(wordService.getAllDictWords());
} catch (Exception e) {
log.error("reloadKnowledge error", e);
}
log.debug("reloadKnowledge end");
}
}

View File

@@ -0,0 +1,47 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.server.service.impl.DictWordService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@Order(2)
public class DictWordLoadStartedListener implements CommandLineRunner {
@Autowired
private DictWordService dictWordService;
@Override
public void run(String... args) {
updateKnowledgeDimValue();
}
public void updateKnowledgeDimValue() {
try {
log.debug("ApplicationStartedInit start");
dictWordService.loadDictWord();
log.debug("ApplicationStartedInit end");
} catch (Exception e) {
log.error("ApplicationStartedInit error", e);
}
}
/***
* reload knowledge task
*/
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
public void reloadKnowledge() {
log.debug("reloadKnowledge start");
try {
dictWordService.reloadDictWord();
} catch (Exception e) {
log.error("reloadKnowledge error", e);
}
log.debug("reloadKnowledge end");
}
}

View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.core.chat.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
@@ -20,13 +21,33 @@ import java.util.stream.Collectors;
@Service
@Slf4j
public class WordService {
public class DictWordService {
@Autowired
private SchemaService schemaService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
private List<DictWord> preDictWords = new ArrayList<>();
public void loadDictWord() {
List<DictWord> dictWords = getAllDictWords();
setPreDictWords(dictWords);
knowledgeBaseService.reloadAllData(dictWords);
}
public void reloadDictWord() {
List<DictWord> dictWords = getAllDictWords();
List<DictWord> preDictWords = getPreDictWords();
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
log.debug("dictWords has not changed, reloadKnowledge end");
return;
}
log.info("dictWords has changed");
setPreDictWords(dictWords);
knowledgeBaseService.updateOnlineKnowledge(getAllDictWords());
}
public List<DictWord> getAllDictWords() {
SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema());

View File

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TagObjectService;
import com.tencent.supersonic.headless.server.service.TermService;
import com.tencent.supersonic.headless.server.service.impl.DictWordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -86,6 +87,8 @@ public abstract class S2BaseDemo implements CommandLineRunner {
protected SysParameterService sysParameterService;
@Autowired
protected CanvasService canvasService;
@Autowired
protected DictWordService dictWordService;
@Value("${s2.demo.names:S2VisitsDemo}")
protected List<String> demoList;
@Value("${s2.demo.enableLLM:true}")

View File

@@ -13,7 +13,6 @@ import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
import com.tencent.supersonic.chat.server.agent.RuleParserTool;
import com.tencent.supersonic.chat.server.plugin.Plugin;
import com.tencent.supersonic.chat.server.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.server.plugin.build.ParamOption;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import com.tencent.supersonic.common.pojo.JoinCondition;
import com.tencent.supersonic.common.pojo.ModelRela;
@@ -111,9 +110,12 @@ public class S2VisitsDemo extends S2BaseDemo {
//create terms and plugin
addTerm(s2Domain);
addTerm_1(s2Domain);
addPlugin(s2DataSet, userDimension, userModel);
addPlugin(s2DataSet);
addSysParameter();
//load dict word
loadDictWord();
//create agent
Integer agentId = addAgent(s2DataSet.getId());
addSampleChats(agentId);
@@ -531,8 +533,7 @@ public class S2VisitsDemo extends S2BaseDemo {
authService.addOrUpdateAuthGroup(authGroupReq);
}
private void addPlugin(DataSetResp s2DataSet, DimensionResp userDimension,
ModelResp userModel) {
private void addPlugin(DataSetResp s2DataSet) {
Plugin plugin1 = new Plugin();
plugin1.setType("WEB_PAGE");
plugin1.setDataSetList(Arrays.asList(s2DataSet.getId()));
@@ -545,13 +546,7 @@ public class S2VisitsDemo extends S2BaseDemo {
plugin1.setParseModeConfig(JSONObject.toJSONString(pluginParseConfig));
WebBase webBase = new WebBase();
webBase.setUrl("www.yourbi.com");
ParamOption paramOption = new ParamOption();
paramOption.setKey("name");
paramOption.setParamType(ParamOption.ParamType.SEMANTIC);
paramOption.setElementId(userDimension.getId());
paramOption.setModelId(userModel.getId());
List<ParamOption> paramOptions = Arrays.asList(paramOption);
webBase.setParamOptions(paramOptions);
webBase.setParamOptions(Lists.newArrayList());
plugin1.setConfig(JsonUtil.toString(webBase));
pluginService.createPlugin(plugin1, user);
}
@@ -565,4 +560,8 @@ public class S2VisitsDemo extends S2BaseDemo {
return tagObjectService.create(tagObjectReq, user);
}
private void loadDictWord() {
dictWordService.loadDictWord();
}
}

View File

@@ -60,14 +60,12 @@ com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor,\
com.tencent.supersonic.chat.server.processor.parse.ParseResultFormatProcessor
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor,\
com.tencent.supersonic.chat.server.processor.execute.QueryResultFormatProcessor
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore

View File

@@ -61,7 +61,7 @@ s2:
embedding:
url: ${s2.pyllm.url}
persistent:
path: /tmp
path: d://
demo:
names: S2VisitsDemo,S2ArtistDemo