mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 11:07:06 +00:00
[improvement][chat]Refactor certain packages to make code cleaner
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
package com.tencent.supersonic.chat.agent;
|
||||
|
||||
import com.tencent.supersonic.chat.parser.llm.analytics.MetricOption;
|
||||
import com.tencent.supersonic.chat.parser.analytics.MetricOption;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.corrector;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.parser.llm.s2sql.ParseResult;
|
||||
import com.tencent.supersonic.chat.parser.sql.llm.ParseResult;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
|
||||
@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.parser.llm.s2sql.S2SQLDateHelper;
|
||||
import com.tencent.supersonic.chat.parser.sql.llm.S2SqlDateHelper;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
@@ -76,7 +76,7 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL);
|
||||
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|
||||
String currentDate = S2SQLDateHelper.getReferenceDate(semanticParseInfo.getModelId());
|
||||
String currentDate = S2SqlDateHelper.getReferenceDate(semanticParseInfo.getModelId());
|
||||
if (StringUtils.isNotBlank(currentDate)) {
|
||||
correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL);
|
||||
correctS2SQL = SqlParserAddHelper.addWhere(
|
||||
|
||||
@@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
@@ -26,8 +26,7 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Query type parser, determine if the query is a metric query, an entity query,
|
||||
* or another type of query.
|
||||
* QueryTypeParser resolves query type as either METRIC or TAG, or OTHER.
|
||||
*/
|
||||
@Slf4j
|
||||
public class QueryTypeParser implements SemanticParser {
|
||||
@@ -55,7 +54,7 @@ public class QueryTypeParser implements SemanticParser {
|
||||
}
|
||||
//1. entity queryType
|
||||
Set<Long> modelIds = parseInfo.getModel().getModelIds();
|
||||
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof S2SQLQuery) {
|
||||
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
|
||||
//If all the fields in the SELECT statement are of tag type.
|
||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL());
|
||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -20,7 +20,7 @@ public class SatisfactionChecker {
|
||||
// check all the parse info in candidate
|
||||
public static boolean check(QueryContext queryContext) {
|
||||
for (SemanticQuery query : queryContext.getCandidateQueries()) {
|
||||
if (query.getQueryMode().equals(S2SQLQuery.QUERY_MODE)) {
|
||||
if (query.getQueryMode().equals(LLMSqlQuery.QUERY_MODE)) {
|
||||
continue;
|
||||
}
|
||||
if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.analytics;
|
||||
package com.tencent.supersonic.chat.parser.analytics;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Sets;
|
||||
@@ -17,7 +17,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.analytics.MetricAnalyzeQuery;
|
||||
import com.tencent.supersonic.chat.service.AgentService;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
@@ -69,7 +69,7 @@ public class MetricAnalyzeParser implements SemanticParser {
|
||||
|
||||
private void buildQuery(Long modelId, QueryContext queryContext,
|
||||
List<Long> metricIds, List<SchemaElementMatch> schemaElementMatches, String toolName) {
|
||||
LLMSemanticQuery metricInterpretQuery = QueryManager.createLLMQuery(MetricInterpretQuery.QUERY_MODE);
|
||||
LLMSemanticQuery metricInterpretQuery = QueryManager.createLLMQuery(MetricAnalyzeQuery.QUERY_MODE);
|
||||
Set<SchemaElement> metrics = getMetrics(metricIds, modelId);
|
||||
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, queryContext.getRequest(),
|
||||
metrics, schemaElementMatches, toolName);
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.analytics;
|
||||
package com.tencent.supersonic.chat.parser.analytics;
|
||||
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.tencent.supersonic.chat.parser;
|
||||
package com.tencent.supersonic.chat.parser.plugin;
|
||||
|
||||
public enum ParseMode {
|
||||
|
||||
RULE,
|
||||
EMBEDDING_RECALL,
|
||||
FUNCTION_CALL;
|
||||
|
||||
@@ -4,7 +4,7 @@ import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.parser.PythonLLMProxy;
|
||||
import com.tencent.supersonic.chat.parser.LLMProxy;
|
||||
import com.tencent.supersonic.chat.parser.ParseMode;
|
||||
import com.tencent.supersonic.chat.parser.plugin.ParseMode;
|
||||
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
|
||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
@@ -23,7 +23,7 @@ import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class EmbeddingBasedParser extends PluginParser {
|
||||
public class EmbeddingRecallParser extends PluginParser {
|
||||
|
||||
protected LLMProxy llmInterpreter = ComponentFactory.getLLMProxy();
|
||||
|
||||
@@ -71,7 +71,7 @@ public class EmbeddingBasedParser extends PluginParser {
|
||||
public List<RecallRetrieval> embeddingRecall(String embeddingText) {
|
||||
try {
|
||||
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
|
||||
EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText);
|
||||
RecallRetrievalResp embeddingResp = pluginManager.recognize(embeddingText);
|
||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||
if (!CollectionUtils.isEmpty(embeddingRetrievals)) {
|
||||
embeddingRetrievals = embeddingRetrievals.stream().sorted(Comparator.comparingDouble(o ->
|
||||
@@ -6,7 +6,7 @@ import lombok.Data;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class EmbeddingResp {
|
||||
public class RecallRetrievalResp {
|
||||
|
||||
private String query;
|
||||
|
||||
@@ -3,13 +3,13 @@ package com.tencent.supersonic.chat.parser.plugin.function;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.parser.PythonLLMProxy;
|
||||
import com.tencent.supersonic.chat.parser.LLMProxy;
|
||||
import com.tencent.supersonic.chat.parser.ParseMode;
|
||||
import com.tencent.supersonic.chat.parser.plugin.ParseMode;
|
||||
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
|
||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
|
||||
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
||||
import com.tencent.supersonic.chat.service.PluginService;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
@@ -25,7 +25,7 @@ import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class FunctionBasedParser extends PluginParser {
|
||||
public class FunctionCallParser extends PluginParser {
|
||||
|
||||
protected LLMProxy llmInterpreter = ComponentFactory.getLLMProxy();
|
||||
|
||||
@@ -97,7 +97,7 @@ public class FunctionBasedParser extends PluginParser {
|
||||
log.info("user decide Model:{}", modelId);
|
||||
List<Plugin> plugins = getPluginList(queryContext);
|
||||
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
|
||||
if (S2SQLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
|
||||
if (LLMSqlQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
|
||||
return false;
|
||||
}
|
||||
if (plugin.getParseModeConfig() == null) {
|
||||
@@ -1,30 +0,0 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* RuleBasedParser acts as a container that incorporates a group of
|
||||
* rule-based semantic parsers.
|
||||
*/
|
||||
@Slf4j
|
||||
public class RuleBasedParser implements SemanticParser {
|
||||
|
||||
private static List<SemanticParser> ruleParsers = Arrays.asList(
|
||||
new QueryModeParser(),
|
||||
new ContextInheritParser(),
|
||||
new AgentCheckParser(),
|
||||
new TimeRangeParser(),
|
||||
new AggregateTypeParser()
|
||||
);
|
||||
|
||||
@Override
|
||||
public void parse(QueryContext queryContext, ChatContext chatContext) {
|
||||
ruleParsers.stream().forEach(p -> p.parse(queryContext, chatContext));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.agent.AgentToolType;
|
||||
import com.tencent.supersonic.chat.agent.NL2SQLTool;
|
||||
@@ -63,11 +63,11 @@ public class LLMRequestService {
|
||||
public boolean check(QueryContext queryCtx) {
|
||||
QueryReq request = queryCtx.getRequest();
|
||||
if (StringUtils.isEmpty(llmParserConfig.getUrl())) {
|
||||
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2SQLParser.class, llmParserConfig);
|
||||
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMSqlParser.class, llmParserConfig);
|
||||
return true;
|
||||
}
|
||||
if (SatisfactionChecker.check(queryCtx)) {
|
||||
log.info("skip {}, queryText:{}", LLMS2SQLParser.class, request.getQueryText());
|
||||
log.info("skip {}, queryText:{}", LLMSqlParser.class, request.getQueryText());
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -134,7 +134,7 @@ public class LLMRequestService {
|
||||
}
|
||||
llmReq.setLinking(linking);
|
||||
|
||||
String currentDate = S2SQLDateHelper.getReferenceDate(firstModelId);
|
||||
String currentDate = S2SqlDateHelper.getReferenceDate(firstModelId);
|
||||
if (StringUtils.isEmpty(currentDate)) {
|
||||
currentDate = DateUtils.getBeforeDate(0);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.agent.NL2SQLTool;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserEqualHelper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -25,7 +25,7 @@ public class LLMResponseService {
|
||||
if (Objects.isNull(weight)) {
|
||||
weight = 0D;
|
||||
}
|
||||
LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(S2SQLQuery.QUERY_MODE);
|
||||
LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE);
|
||||
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||
parseInfo.setModel(parseResult.getModelCluster());
|
||||
NL2SQLTool commonAgentTool = parseResult.getCommonAgentTool();
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.agent.NL2SQLTool;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
@@ -21,7 +21,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
@Slf4j
|
||||
public class LLMS2SQLParser implements SemanticParser {
|
||||
public class LLMSqlParser implements SemanticParser {
|
||||
|
||||
@Override
|
||||
public void parse(QueryContext queryCtx, ChatContext chatCtx) {
|
||||
@@ -41,7 +41,7 @@ public class LLMS2SQLParser implements SemanticParser {
|
||||
//3.get agent tool and determine whether to skip this parser.
|
||||
NL2SQLTool commonAgentTool = requestService.getParserTool(request, modelCluster.getModelIds());
|
||||
if (Objects.isNull(commonAgentTool)) {
|
||||
log.info("no tool in this agent, skip {}", LLMS2SQLParser.class);
|
||||
log.info("no tool in this agent, skip {}", LLMSqlParser.class);
|
||||
return;
|
||||
}
|
||||
//4.construct a request, call the API for the large model, and retrieve the results.
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.agent.NL2SQLTool;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
|
||||
@@ -11,7 +11,7 @@ import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
public class S2SQLDateHelper {
|
||||
public class S2SqlDateHelper {
|
||||
|
||||
public static String getReferenceDate(Long modelId) {
|
||||
String defaultDate = DateUtils.getBeforeDate(0);
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
package com.tencent.supersonic.chat.parser.sql.rule;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
package com.tencent.supersonic.chat.parser.sql.rule;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG;
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
package com.tencent.supersonic.chat.parser.sql.rule;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
package com.tencent.supersonic.chat.parser.sql.rule;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
@@ -8,14 +8,22 @@ import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
||||
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* QueryModeParser resolves a specific query mode according to co-appearance
|
||||
* RuleSqlParser resolves a specific SemanticQuery according to co-appearance
|
||||
* of certain schema element types.
|
||||
*/
|
||||
@Slf4j
|
||||
public class QueryModeParser implements SemanticParser {
|
||||
public class RuleSqlParser implements SemanticParser {
|
||||
|
||||
private static List<SemanticParser> auxiliaryParsers = Arrays.asList(
|
||||
new ContextInheritParser(),
|
||||
new AgentCheckParser(),
|
||||
new TimeRangeParser(),
|
||||
new AggregateTypeParser()
|
||||
);
|
||||
|
||||
@Override
|
||||
public void parse(QueryContext queryContext, ChatContext chatContext) {
|
||||
@@ -29,6 +37,7 @@ public class QueryModeParser implements SemanticParser {
|
||||
queryContext.getCandidateQueries().add(query);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auxiliaryParsers.stream().forEach(p -> p.parse(queryContext, chatContext));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.parser.rule;
|
||||
package com.tencent.supersonic.chat.parser.sql.rule;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.plugin;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.parser.ParseMode;
|
||||
import com.tencent.supersonic.chat.parser.plugin.ParseMode;
|
||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||
import lombok.Data;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
@@ -12,7 +12,7 @@ import com.tencent.supersonic.chat.agent.Agent;
|
||||
import com.tencent.supersonic.chat.agent.AgentToolType;
|
||||
import com.tencent.supersonic.chat.agent.PluginTool;
|
||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrievalResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
|
||||
import com.tencent.supersonic.chat.plugin.event.PluginAddEvent;
|
||||
import com.tencent.supersonic.chat.plugin.event.PluginDelEvent;
|
||||
@@ -165,7 +165,7 @@ public class PluginManager {
|
||||
requestEmbeddingPluginAdd(convert(plugins));
|
||||
}
|
||||
|
||||
public EmbeddingResp recognize(String embeddingText) {
|
||||
public RecallRetrievalResp recognize(String embeddingText) {
|
||||
String url = embeddingConfig.getUrl() + embeddingConfig.getRecognizePath() + "?n_results="
|
||||
+ embeddingConfig.getNResult();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
@@ -176,14 +176,14 @@ public class PluginManager {
|
||||
String jsonBody = JSONObject.toJSONString(Lists.newArrayList(embeddingText));
|
||||
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
|
||||
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
|
||||
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
|
||||
ResponseEntity<List<RecallRetrievalResp>> embeddingResponseEntity =
|
||||
restTemplate.exchange(requestUrl, HttpMethod.POST, entity,
|
||||
new ParameterizedTypeReference<List<EmbeddingResp>>() {
|
||||
new ParameterizedTypeReference<List<RecallRetrievalResp>>() {
|
||||
});
|
||||
log.info("[embedding] recognize result body:{}", embeddingResponseEntity);
|
||||
List<EmbeddingResp> embeddingResps = embeddingResponseEntity.getBody();
|
||||
List<RecallRetrievalResp> embeddingResps = embeddingResponseEntity.getBody();
|
||||
if (CollectionUtils.isNotEmpty(embeddingResps)) {
|
||||
for (EmbeddingResp embeddingResp : embeddingResps) {
|
||||
for (RecallRetrievalResp embeddingResp : embeddingResps) {
|
||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
||||
embeddingRetrieval.setId(getPluginIdFromEmbeddingId(embeddingRetrieval.getId()));
|
||||
|
||||
@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.analytics.MetricAnalyzeQuery;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -16,7 +16,11 @@ import org.springframework.util.CollectionUtils;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class EntityInfoProcessor implements ResponseProcessor {
|
||||
/**
|
||||
* EntityInfoProcessor fills core attributes of an entity so that
|
||||
* users get to know which entity is parsed out.
|
||||
*/
|
||||
public class EntityInfoProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
@@ -30,7 +34,7 @@ public class EntityInfoProcessor implements ResponseProcessor {
|
||||
selectedParses.forEach(parseInfo -> {
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
if (QueryManager.containsPluginQuery(queryMode)
|
||||
|| MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
|
||||
|| MetricAnalyzeQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
|
||||
return;
|
||||
}
|
||||
//1. set entity info
|
||||
|
||||
@@ -27,10 +27,10 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* MetricCheckProcessor used to verify whether the dimensions
|
||||
* MetricCheckProcessor verifies whether the dimensions
|
||||
* involved in the query in metric mode can drill down on the metric.
|
||||
*/
|
||||
public class MetricCheckProcessor implements ResponseProcessor {
|
||||
public class MetricCheckProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
|
||||
@@ -33,10 +33,11 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* update parse info from correct sql
|
||||
*/
|
||||
* ParseInfoProcessor extracts structured info from S2SQL so that
|
||||
* users get to know the details.
|
||||
**/
|
||||
@Slf4j
|
||||
public class ParseInfoProcessor implements ResponseProcessor {
|
||||
public class ParseInfoProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
@@ -71,6 +72,7 @@ public class ParseInfoProcessor implements ResponseProcessor {
|
||||
} catch (Exception e) {
|
||||
log.error("set dateInfo error :", e);
|
||||
}
|
||||
|
||||
//set filter
|
||||
Set<Long> modelIds = parseInfo.getModel().getModelIds();
|
||||
try {
|
||||
@@ -80,6 +82,7 @@ public class ParseInfoProcessor implements ResponseProcessor {
|
||||
} catch (Exception e) {
|
||||
log.error("set dimensionFilter error :", e);
|
||||
}
|
||||
|
||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
||||
if (Objects.isNull(semanticSchema)) {
|
||||
return;
|
||||
|
||||
@@ -4,9 +4,9 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
|
||||
/**
|
||||
* A response processor wraps things up before responding to users.
|
||||
* A ParseResultProcessor wraps things up before returning results to users.
|
||||
*/
|
||||
public interface ResponseProcessor {
|
||||
public interface ParseResultProcessor {
|
||||
|
||||
void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);
|
||||
|
||||
@@ -15,10 +15,11 @@ import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Rank queries by score.
|
||||
*/
|
||||
* QueryRankProcessor ranks candidate parsing results based on
|
||||
* a heuristic scoring algorithm and then takes topN.
|
||||
**/
|
||||
@Slf4j
|
||||
public class QueryRankProcessor implements ResponseProcessor {
|
||||
public class QueryRankProcessor implements ParseResultProcessor {
|
||||
|
||||
private static final int candidateTopSize = 5;
|
||||
|
||||
|
||||
@@ -12,8 +12,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* RespBuildProcessor fill response object with parsing results.
|
||||
**/
|
||||
@Slf4j
|
||||
public class RespBuildProcessor implements ResponseProcessor {
|
||||
public class RespBuildProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
|
||||
@@ -13,7 +13,11 @@ import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SqlInfoProcessor implements ResponseProcessor {
|
||||
/**
|
||||
* SqlInfoProcessor adds S2SQL to the parsing results so that
|
||||
* technical users could verify SQL by themselves.
|
||||
**/
|
||||
public class SqlInfoProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
|
||||
@@ -5,8 +5,11 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* TimeCostProcessor adds time cost of parsing.
|
||||
**/
|
||||
@Slf4j
|
||||
public class TimeCostProcessor implements ResponseProcessor {
|
||||
public class TimeCostProcessor implements ParseResultProcessor {
|
||||
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,25 +1,28 @@
|
||||
package com.tencent.supersonic.chat.responder;
|
||||
package com.tencent.supersonic.chat.query;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.analytics.MetricAnalyzeQuery;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
|
||||
/**
|
||||
* EntityInfoQueryResponder fills core attributes of an entity so that
|
||||
* users get to know which entity is returned.
|
||||
*/
|
||||
public class EntityInfoQueryResponder implements QueryResponder {
|
||||
|
||||
@Override
|
||||
public void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {
|
||||
public void fillInfo(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {
|
||||
if (semanticParseInfo == null) {
|
||||
return;
|
||||
}
|
||||
String queryMode = semanticParseInfo.getQueryMode();
|
||||
if (QueryManager.containsPluginQuery(queryMode)
|
||||
|| MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
|
||||
|| MetricAnalyzeQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
|
||||
return;
|
||||
}
|
||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
||||
@@ -1,11 +1,14 @@
|
||||
package com.tencent.supersonic.chat.responder;
|
||||
package com.tencent.supersonic.chat.query;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
|
||||
/**
|
||||
* A query responder fill some auxiliary information into query results.
|
||||
*/
|
||||
public interface QueryResponder {
|
||||
|
||||
void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq);
|
||||
void fillInfo(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq);
|
||||
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.responder;
|
||||
package com.tencent.supersonic.chat.query;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
@@ -22,12 +22,15 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* SimilarMetricQueryResponder fills recommended metrics based on embedding similarity.
|
||||
*/
|
||||
public class SimilarMetricQueryResponder implements QueryResponder {
|
||||
|
||||
private static final int METRIC_RECOMMEND_SIZE = 5;
|
||||
|
||||
@Override
|
||||
public void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {
|
||||
public void fillInfo(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {
|
||||
fillSimilarMetric(queryResult.getChatContext());
|
||||
}
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
package com.tencent.supersonic.chat.query.llm;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.query.BaseSemanticQuery;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public abstract class LLMSemanticQuery extends BaseSemanticQuery {
|
||||
|
||||
@Override
|
||||
public void initS2Sql(User user) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.query.llm.interpret;
|
||||
package com.tencent.supersonic.chat.query.llm.analytics;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.query.llm.interpret;
|
||||
package com.tencent.supersonic.chat.query.llm.analytics;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.query.llm.interpret;
|
||||
package com.tencent.supersonic.chat.query.llm.analytics;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
@@ -36,13 +36,13 @@ import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MetricInterpretQuery extends LLMSemanticQuery {
|
||||
public class MetricAnalyzeQuery extends LLMSemanticQuery {
|
||||
|
||||
|
||||
public static final String QUERY_MODE = "METRIC_INTERPRET";
|
||||
|
||||
|
||||
public MetricInterpretQuery() {
|
||||
public MetricAnalyzeQuery() {
|
||||
QueryManager.register(this);
|
||||
}
|
||||
|
||||
@@ -22,12 +22,12 @@ import java.util.Objects;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class S2SQLQuery extends LLMSemanticQuery {
|
||||
public class LLMSqlQuery extends LLMSemanticQuery {
|
||||
|
||||
public static final String QUERY_MODE = "LLM_S2SQL";
|
||||
protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
|
||||
|
||||
public S2SQLQuery() {
|
||||
public LLMSqlQuery() {
|
||||
QueryManager.register(this);
|
||||
}
|
||||
|
||||
@@ -48,15 +48,15 @@ public class WebPageQuery extends PluginSemanticQuery {
|
||||
Map<String, Object> properties = parseInfo.getProperties();
|
||||
PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)),
|
||||
PluginParseResult.class);
|
||||
WebPageResponse webPageResponse = buildResponse(pluginParseResult);
|
||||
WebPageResp webPageResponse = buildResponse(pluginParseResult);
|
||||
queryResult.setResponse(webPageResponse);
|
||||
queryResult.setQueryState(QueryState.SUCCESS);
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
protected WebPageResponse buildResponse(PluginParseResult pluginParseResult) {
|
||||
protected WebPageResp buildResponse(PluginParseResult pluginParseResult) {
|
||||
Plugin plugin = pluginParseResult.getPlugin();
|
||||
WebPageResponse webPageResponse = new WebPageResponse();
|
||||
WebPageResp webPageResponse = new WebPageResp();
|
||||
webPageResponse.setName(plugin.getName());
|
||||
webPageResponse.setPluginId(plugin.getId());
|
||||
webPageResponse.setPluginType(plugin.getType());
|
||||
|
||||
@@ -5,7 +5,7 @@ import lombok.Data;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class WebPageResponse {
|
||||
public class WebPageResp {
|
||||
|
||||
private Long pluginId;
|
||||
|
||||
@@ -56,7 +56,7 @@ public class WebServiceQuery extends PluginSemanticQuery {
|
||||
Map<String, Object> properties = parseInfo.getProperties();
|
||||
PluginParseResult pluginParseResult = JsonUtil.toObject(
|
||||
JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
|
||||
WebServiceResponse webServiceResponse = buildResponse(pluginParseResult);
|
||||
WebServiceResp webServiceResponse = buildResponse(pluginParseResult);
|
||||
Object object = webServiceResponse.getResult();
|
||||
// in order to show webServiceQuery result int frontend conveniently,
|
||||
// webServiceResponse result format is consistent with queryByStruct result.
|
||||
@@ -72,8 +72,8 @@ public class WebServiceQuery extends PluginSemanticQuery {
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
protected WebServiceResponse buildResponse(PluginParseResult pluginParseResult) {
|
||||
WebServiceResponse webServiceResponse = new WebServiceResponse();
|
||||
protected WebServiceResp buildResponse(PluginParseResult pluginParseResult) {
|
||||
WebServiceResp webServiceResponse = new WebServiceResp();
|
||||
Plugin plugin = pluginParseResult.getPlugin();
|
||||
WebBase webBase = JsonUtil.toObject(plugin.getConfig(), WebBase.class);
|
||||
webServiceResponse.setWebBase(webBase);
|
||||
|
||||
@@ -5,7 +5,7 @@ import lombok.Data;
|
||||
|
||||
|
||||
@Data
|
||||
public class WebServiceResponse {
|
||||
public class WebServiceResp {
|
||||
|
||||
private WebBase webBase;
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
|
||||
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
|
||||
import com.tencent.supersonic.chat.service.KnowledgeService;
|
||||
import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
@@ -30,7 +30,7 @@ import org.springframework.web.bind.annotation.PutMapping;
|
||||
public class KnowledgeController {
|
||||
|
||||
@Autowired
|
||||
private ChatKnowledgeService knowledgeService;
|
||||
private KnowledgeService knowledgeService;
|
||||
|
||||
@Autowired
|
||||
private ApplicationStartedListener applicationStartedListener;
|
||||
|
||||
@@ -10,7 +10,7 @@ import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ChatKnowledgeService {
|
||||
public interface KnowledgeService {
|
||||
Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
|
||||
|
||||
Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
|
||||
@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
|
||||
import com.tencent.supersonic.chat.config.DefaultMetric;
|
||||
import com.tencent.supersonic.chat.config.Dim4Dict;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
|
||||
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
|
||||
import com.tencent.supersonic.chat.service.KnowledgeService;
|
||||
import com.tencent.supersonic.chat.utils.DictMetaHelper;
|
||||
import com.tencent.supersonic.chat.utils.DictQueryHelper;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
@@ -42,7 +42,7 @@ import java.util.HashMap;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ChatKnowledgeServiceImpl implements ChatKnowledgeService {
|
||||
public class ChatKnowledgeServiceImpl implements KnowledgeService {
|
||||
|
||||
private final DictMetaHelper metaUtils;
|
||||
private final DictQueryHelper dictQueryHelper;
|
||||
|
||||
@@ -28,11 +28,11 @@ import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
|
||||
import com.tencent.supersonic.chat.processor.ResponseProcessor;
|
||||
import com.tencent.supersonic.chat.processor.ParseResultProcessor;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
|
||||
import com.tencent.supersonic.chat.responder.QueryResponder;
|
||||
import com.tencent.supersonic.chat.query.QueryResponder;
|
||||
import com.tencent.supersonic.chat.service.ChatService;
|
||||
import com.tencent.supersonic.chat.service.QueryService;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
@@ -111,7 +111,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
|
||||
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
|
||||
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
|
||||
private List<ResponseProcessor> responseProcessors = ComponentFactory.getPostProcessors();
|
||||
private List<ParseResultProcessor> responseProcessors = ComponentFactory.getPostProcessors();
|
||||
private List<QueryResponder> executeResponders = ComponentFactory.getExecuteResponders();
|
||||
private List<SemanticCorrector> semanticCorrectors = ComponentFactory.getSemanticCorrectors();
|
||||
|
||||
@@ -208,7 +208,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
chatCtx.setQueryText(queryReq.getQueryText());
|
||||
chatCtx.setUser(queryReq.getUser().getName());
|
||||
for (QueryResponder executeResponder : executeResponders) {
|
||||
executeResponder.fillResponse(queryResult, parseInfo, queryReq);
|
||||
executeResponder.fillInfo(queryResult, parseInfo, queryReq);
|
||||
}
|
||||
chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);
|
||||
} else {
|
||||
@@ -277,7 +277,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
//replace metrics
|
||||
log.info("llm begin replace metrics!");
|
||||
replaceMetrics(parseInfo, metrics);
|
||||
} else if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
|
||||
} else if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
|
||||
log.info("llm begin revise filters!");
|
||||
String correctorSql = reviseCorrectS2SQL(queryData, parseInfo);
|
||||
parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql);
|
||||
@@ -536,7 +536,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
|
||||
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
|
||||
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
||||
if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
|
||||
if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
|
||||
return parseInfo;
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {
|
||||
|
||||
@@ -5,9 +5,9 @@ import com.tencent.supersonic.chat.api.component.SemanticCorrector;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.parser.LLMProxy;
|
||||
import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
|
||||
import com.tencent.supersonic.chat.processor.ResponseProcessor;
|
||||
import com.tencent.supersonic.chat.responder.QueryResponder;
|
||||
import com.tencent.supersonic.chat.parser.sql.llm.ModelResolver;
|
||||
import com.tencent.supersonic.chat.processor.ParseResultProcessor;
|
||||
import com.tencent.supersonic.chat.query.QueryResponder;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||
import java.util.ArrayList;
|
||||
@@ -22,7 +22,7 @@ public class ComponentFactory {
|
||||
private static SemanticInterpreter semanticInterpreter;
|
||||
|
||||
private static LLMProxy llmProxy;
|
||||
private static List<ResponseProcessor> responseProcessors = new ArrayList<>();
|
||||
private static List<ParseResultProcessor> responseProcessors = new ArrayList<>();
|
||||
private static List<QueryResponder> executeResponders = new ArrayList<>();
|
||||
private static ModelResolver modelResolver;
|
||||
|
||||
@@ -39,8 +39,8 @@ public class ComponentFactory {
|
||||
semanticCorrectors) : semanticCorrectors;
|
||||
}
|
||||
|
||||
public static List<ResponseProcessor> getPostProcessors() {
|
||||
return CollectionUtils.isEmpty(responseProcessors) ? init(ResponseProcessor.class,
|
||||
public static List<ParseResultProcessor> getPostProcessors() {
|
||||
return CollectionUtils.isEmpty(responseProcessors) ? init(ParseResultProcessor.class,
|
||||
responseProcessors) : responseProcessors;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrievalResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
@@ -82,15 +82,15 @@ public class SolvedQueryManager {
|
||||
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
|
||||
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
|
||||
ResponseEntity<List<RecallRetrievalResp>> embeddingResponseEntity =
|
||||
restTemplate.exchange(requestUrl, HttpMethod.POST, entity,
|
||||
new ParameterizedTypeReference<List<EmbeddingResp>>() {
|
||||
new ParameterizedTypeReference<List<RecallRetrievalResp>>() {
|
||||
});
|
||||
log.info("[embedding] recognize result body:{}", embeddingResponseEntity);
|
||||
List<EmbeddingResp> embeddingResps = embeddingResponseEntity.getBody();
|
||||
List<RecallRetrievalResp> embeddingResps = embeddingResponseEntity.getBody();
|
||||
Set<String> querySet = new HashSet<>();
|
||||
if (CollectionUtils.isNotEmpty(embeddingResps)) {
|
||||
for (EmbeddingResp embeddingResp : embeddingResps) {
|
||||
for (RecallRetrievalResp embeddingResp : embeddingResps) {
|
||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
||||
if (queryText.equalsIgnoreCase(embeddingRetrieval.getQuery())) {
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.application.parser;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.parser.rule.TimeRangeParser;
|
||||
import com.tencent.supersonic.chat.parser.sql.rule.TimeRangeParser;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.application.parser.aggregate;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import com.tencent.supersonic.chat.parser.rule.AggregateTypeParser;
|
||||
import com.tencent.supersonic.chat.parser.sql.rule.AggregateTypeParser;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.tencent.supersonic.chat.parser.llm.s2sql;
|
||||
|
||||
import com.tencent.supersonic.chat.parser.sql.llm.LLMResponseService;
|
||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -6,10 +6,10 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
|
||||
com.tencent.supersonic.chat.mapper.EntityMapper
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticParser=\
|
||||
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.rule.RuleSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.llm.LLMSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingRecallParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionCallParser, \
|
||||
com.tencent.supersonic.chat.parser.QueryTypeParser
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
@@ -25,8 +25,8 @@ com.tencent.supersonic.chat.parser.LLMProxy=\
|
||||
com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
|
||||
com.tencent.supersonic.knowledge.semantic.RemoteSemanticInterpreter
|
||||
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
|
||||
com.tencent.supersonic.chat.parser.sql.llm.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.sql.llm.HeuristicModelResolver
|
||||
|
||||
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
|
||||
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor
|
||||
@@ -43,6 +43,6 @@ com.tencent.supersonic.chat.postprocessor.PostProcessor=\
|
||||
com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
|
||||
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
|
||||
|
||||
com.tencent.supersonic.chat.responder.QueryResponder=\
|
||||
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder
|
||||
com.tencent.supersonic.chat.query.QueryResponder=\
|
||||
com.tencent.supersonic.chat.query.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.query.SimilarMetricQueryResponder
|
||||
@@ -7,10 +7,10 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
|
||||
com.tencent.supersonic.chat.mapper.ModelClusterMapper
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticParser=\
|
||||
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.rule.RuleSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.llm.LLMSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingRecallParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionCallParser, \
|
||||
com.tencent.supersonic.chat.parser.QueryTypeParser
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
@@ -21,7 +21,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
com.tencent.supersonic.chat.corrector.HavingCorrector, \
|
||||
com.tencent.supersonic.chat.corrector.FromCorrector
|
||||
|
||||
com.tencent.supersonic.chat.processor.ResponseProcessor=\
|
||||
com.tencent.supersonic.chat.processor.ParseResultProcessor=\
|
||||
com.tencent.supersonic.chat.processor.MetricCheckProcessor, \
|
||||
com.tencent.supersonic.chat.processor.ParseInfoProcessor, \
|
||||
com.tencent.supersonic.chat.processor.QueryRankProcessor, \
|
||||
@@ -36,8 +36,8 @@ com.tencent.supersonic.chat.parser.LLMProxy=\
|
||||
com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
|
||||
com.tencent.supersonic.knowledge.semantic.LocalSemanticInterpreter
|
||||
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
|
||||
com.tencent.supersonic.chat.parser.sql.llm.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.sql.llm.HeuristicModelResolver
|
||||
|
||||
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
|
||||
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor
|
||||
@@ -45,6 +45,6 @@ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor
|
||||
com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
|
||||
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
|
||||
|
||||
com.tencent.supersonic.chat.responder.QueryResponder=\
|
||||
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder
|
||||
com.tencent.supersonic.chat.query.QueryResponder=\
|
||||
com.tencent.supersonic.chat.query.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.query.SimilarMetricQueryResponder
|
||||
@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
import com.tencent.supersonic.chat.query.llm.interpret.LLMAnswerResp;
|
||||
import com.tencent.supersonic.chat.query.llm.analytics.LLMAnswerResp;
|
||||
import com.tencent.supersonic.chat.service.AgentService;
|
||||
import com.tencent.supersonic.chat.service.QueryService;
|
||||
import com.tencent.supersonic.util.DataUtils;
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.integration;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrievalResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
import com.tencent.supersonic.chat.service.AgentService;
|
||||
@@ -20,7 +20,7 @@ import static org.mockito.Mockito.when;
|
||||
public class MockConfiguration {
|
||||
|
||||
public static void mockEmbeddingRecognize(PluginManager pluginManager, String text, String id) {
|
||||
EmbeddingResp embeddingResp = new EmbeddingResp();
|
||||
RecallRetrievalResp embeddingResp = new RecallRetrievalResp();
|
||||
RecallRetrieval embeddingRetrieval = new RecallRetrieval();
|
||||
embeddingRetrieval.setId(id);
|
||||
embeddingRetrieval.setPresetId(id);
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
|
||||
import com.tencent.supersonic.chat.query.plugin.webpage.WebPageQuery;
|
||||
import com.tencent.supersonic.chat.query.plugin.webpage.WebPageResponse;
|
||||
import com.tencent.supersonic.chat.query.plugin.webpage.WebPageResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.Assert;
|
||||
import org.junit.runner.RunWith;
|
||||
@@ -22,7 +22,7 @@ public class BasePluginTest {
|
||||
protected void assertPluginRecognizeResult(QueryResult queryResult) {
|
||||
Assert.assertEquals(queryResult.getQueryState(), QueryState.SUCCESS);
|
||||
Assert.assertEquals(queryResult.getQueryMode(), WebPageQuery.QUERY_MODE);
|
||||
WebPageResponse webPageResponse = (WebPageResponse) queryResult.getResponse();
|
||||
WebPageResp webPageResponse = (WebPageResp) queryResult.getResponse();
|
||||
WebBaseResult webPage = webPageResponse.getWebPage();
|
||||
Assert.assertEquals(webPage.getUrl(), "www.yourbi.com");
|
||||
Assert.assertEquals(1, webPage.getParams().size());
|
||||
|
||||
@@ -13,7 +13,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.parser.llm.analytics.MetricOption;
|
||||
import com.tencent.supersonic.chat.parser.analytics.MetricOption;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
|
||||
|
||||
@@ -7,10 +7,10 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
|
||||
com.tencent.supersonic.chat.mapper.ModelClusterMapper
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticParser=\
|
||||
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.rule.RuleSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.sql.llm.LLMSqlParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingRecallParser, \
|
||||
com.tencent.supersonic.chat.parser.plugin.function.FunctionCallParser, \
|
||||
com.tencent.supersonic.chat.parser.QueryTypeParser
|
||||
|
||||
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
@@ -23,8 +23,8 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
|
||||
com.tencent.supersonic.knowledge.semantic.LocalSemanticInterpreter
|
||||
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
|
||||
com.tencent.supersonic.chat.parser.sql.llm.ModelResolver=\
|
||||
com.tencent.supersonic.chat.parser.sql.llm.HeuristicModelResolver
|
||||
|
||||
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
|
||||
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor
|
||||
@@ -42,6 +42,6 @@ com.tencent.supersonic.chat.postprocessor.PostProcessor=\
|
||||
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
|
||||
|
||||
|
||||
com.tencent.supersonic.chat.responder.QueryResponder=\
|
||||
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder
|
||||
com.tencent.supersonic.chat.query.QueryResponder=\
|
||||
com.tencent.supersonic.chat.query.EntityInfoQueryResponder, \
|
||||
com.tencent.supersonic.chat.query.SimilarMetricQueryResponder
|
||||
Reference in New Issue
Block a user