diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java index 4e0f9f4ee..3ad49c784 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java @@ -16,16 +16,6 @@ public class ChatConfigBaseReq { private Long modelId; - /** - * the chatDetailConfig about the model - */ - private ChatDetailConfigReq chatDetailConfig; - - /** - * the chatAggConfig about the model - */ - private ChatAggConfigReq chatAggConfig; - /** * the recommended questions about the model diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java new file mode 100644 index 000000000..0080cef8e --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatExecuteReq.java @@ -0,0 +1,21 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ChatExecuteReq { + private User user; + private Long queryId; + private Integer chatId; + private int parseId; + private String queryText; + private boolean saveAnswer; + +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java similarity index 77% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java index 03e946136..2307043d6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatParseReq.java @@ -1,15 +1,16 @@ package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import lombok.Data; @Data -public class QueryReq { +public class ChatParseReq { private String queryText; private Integer chatId; - private Long dataSetId; + private Integer agentId; private User user; private QueryFilters queryFilters; private boolean saveAnswer = true; - private Integer agentId; + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java index 9b386e3c5..1669b68f2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ShowCaseResp.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.api.pojo.response; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; import lombok.Data; import java.util.List; diff --git a/chat/core/pom.xml b/chat/core/pom.xml deleted file mode 100644 index 2621995b7..000000000 --- a/chat/core/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - chat - com.tencent.supersonic - ${revision} - - 4.0.0 - - chat-core - - - 8 - 8 - - - - - org.springframework - spring-context - - - org.testng - testng - ${org.testng.version} - test - - - org.junit.jupiter - junit-jupiter - test - - - org.apache.commons - commons-compress - ${commons.compress.version} - compile - - - - org.springframework.boot - spring-boot-starter-test - - - junit - junit - test - - - - org.springframework.boot - spring-boot-starter-web - - - - com.alibaba - druid - ${alibaba.druid.version} - - - mysql - mysql-connector-java - - - - com.h2database - h2 - ${h2.version} - - - com.tencent.supersonic - headless-api - ${project.version} - - - com.tencent.supersonic - headless-core - ${project.version} - - - com.tencent.supersonic - chat-api - ${project.version} - compile - - - - com.github.xkzhangsan - xk-time - ${xk.time.version} - - - - org.mockito - mockito-inline - ${mockito-inline.version} - test - - - com.tencent.supersonic - headless-server - ${project.version} - compile - - - - - diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java deleted file mode 100644 index 0e1119cb2..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.tencent.supersonic.chat.core.parser; - -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionPromptGenerator; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; -import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat; -import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGeneration; -import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGenerationFactory; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.common.util.ContextUtils; -import dev.langchain4j.model.chat.ChatLanguageModel; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.util.Objects; - -/** - * LLMProxy based on langchain4j Java version. - */ -@Slf4j -@Component -public class JavaLLMProxy implements LLMProxy { - - private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); - - @Override - public boolean isSkip(QueryContext queryContext) { - ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); - if (Objects.isNull(chatLanguageModel)) { - log.warn("chatLanguageModel is null, skip :{}", JavaLLMProxy.class.getName()); - return true; - } - return false; - } - - public LLMResp query2sql(LLMReq llmReq, Long dataSetId) { - - SqlGeneration sqlGeneration = SqlGenerationFactory.get( - SqlGenerationMode.getMode(llmReq.getSqlGenerationMode())); - String modelName = llmReq.getSchema().getDataSetName(); - LLMResp result = sqlGeneration.generation(llmReq, dataSetId); - result.setQuery(llmReq.getQueryText()); - result.setModelName(modelName); - return result; - } - - @Override - public FunctionResp requestFunction(FunctionReq functionReq) { - - FunctionPromptGenerator promptGenerator = ContextUtils.getBean(FunctionPromptGenerator.class); - - ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); - String functionCallPrompt = promptGenerator.generateFunctionCallPrompt(functionReq.getQueryText(), - functionReq.getPluginConfigs()); - keyPipelineLog.info("functionCallPrompt:{}", functionCallPrompt); - String response = chatLanguageModel.generate(functionCallPrompt); - keyPipelineLog.info("functionCall response:{}", response); - return OutputFormat.functionCallParse(response); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java deleted file mode 100644 index d11cefa77..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.supersonic.chat.core.parser; - -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; - -/** - * LLMProxy encapsulates functions performed by LLMs so that multiple - * orchestration frameworks (e.g. LangChain in python, LangChain4j in java) - * could be used. - */ -public interface LLMProxy { - - boolean isSkip(QueryContext queryContext); - - LLMResp query2sql(LLMReq llmReq, Long dataSetId); - - FunctionResp requestFunction(FunctionReq functionReq); - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java deleted file mode 100644 index 7c8ba3fa0..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; - -import com.tencent.supersonic.chat.core.parser.PythonLLMProxy; -import com.tencent.supersonic.chat.core.parser.plugin.ParseMode; -import com.tencent.supersonic.chat.core.parser.plugin.PluginParser; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginManager; -import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; -import com.tencent.supersonic.chat.core.plugin.PluginRecallResult; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.JsonUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * FunctionCallParser is an implementation of a recall plugin based on FunctionCall - */ -@Slf4j -public class FunctionCallParser extends PluginParser { - - @Override - public boolean checkPreCondition(QueryContext queryContext) { - FunctionCallConfig functionCallConfig = ContextUtils.getBean(FunctionCallConfig.class); - String functionUrl = functionCallConfig.getUrl(); - if (StringUtils.isBlank(functionUrl) && ComponentFactory.getLLMProxy() instanceof PythonLLMProxy) { - log.info("functionUrl:{}, skip function parser, queryText:{}", functionUrl, - queryContext.getQueryText()); - return false; - } - List plugins = getPluginList(queryContext); - return !CollectionUtils.isEmpty(plugins); - } - - @Override - public PluginRecallResult recallPlugin(QueryContext queryContext) { - FunctionResp functionResp = functionCall(queryContext); - if (skipFunction(functionResp)) { - return null; - } - log.info("requestFunction result:{}", functionResp.getToolSelection()); - String toolSelection = functionResp.getToolSelection(); - Plugin plugin = queryContext.getNameToPlugin().get(toolSelection); - if (Objects.isNull(plugin)) { - log.info("pluginOptional is not exist:{}, skip the parse", toolSelection); - return null; - } - plugin.setParseMode(ParseMode.FUNCTION_CALL); - Pair> pluginResolveResult = PluginManager.resolve(plugin, queryContext); - if (pluginResolveResult.getLeft()) { - Set dataSetList = pluginResolveResult.getRight(); - if (CollectionUtils.isEmpty(dataSetList)) { - return null; - } - double score = queryContext.getQueryText().length(); - return PluginRecallResult.builder().plugin(plugin).dataSetIds(dataSetList).score(score).build(); - } - return null; - } - - public FunctionResp functionCall(QueryContext queryContext) { - List pluginToFunctionCall = - getPluginToFunctionCall(queryContext.getDataSetId(), queryContext); - if (CollectionUtils.isEmpty(pluginToFunctionCall)) { - log.info("function call parser, plugin is empty, skip"); - return null; - } - FunctionResp functionResp = new FunctionResp(); - if (pluginToFunctionCall.size() == 1) { - functionResp.setToolSelection(pluginToFunctionCall.iterator().next().getName()); - } else { - FunctionReq functionReq = FunctionReq.builder() - .queryText(queryContext.getQueryText()) - .pluginConfigs(pluginToFunctionCall).build(); - functionResp = ComponentFactory.getLLMProxy().requestFunction(functionReq); - } - return functionResp; - } - - private boolean skipFunction(FunctionResp functionResp) { - return Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection()); - } - - private List getPluginToFunctionCall(Long modelId, QueryContext queryContext) { - log.info("user decide Model:{}", modelId); - List plugins = getPluginList(queryContext); - List functionDOList = plugins.stream().filter(plugin -> { - if (LLMSqlQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) { - return false; - } - if (plugin.getParseModeConfig() == null) { - return false; - } - PluginParseConfig pluginParseConfig = JsonUtil.toObject(plugin.getParseModeConfig(), - PluginParseConfig.class); - if (StringUtils.isBlank(pluginParseConfig.getName())) { - return false; - } - Pair> pluginResolverResult = PluginManager.resolve(plugin, queryContext); - log.info("plugin [{}-{}] resolve: {}", plugin.getId(), plugin.getName(), pluginResolverResult); - if (!pluginResolverResult.getLeft()) { - return false; - } else { - Set resolveModel = pluginResolverResult.getRight(); - if (modelId != null && modelId > 0) { - if (plugin.isContainsAllModel()) { - return true; - } - return resolveModel.contains(modelId); - } - return true; - } - }).map(o -> JsonUtil.toObject(o.getParseModeConfig(), PluginParseConfig.class)).collect(Collectors.toList()); - log.info("PluginToFunctionCall: {}", JsonUtil.toString(functionDOList)); - return functionDOList; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java deleted file mode 100644 index d3866da51..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; - -import com.tencent.supersonic.chat.core.agent.NL2SQLTool; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class ParseResult { - - private Long dataSetId; - - private LLMReq llmReq; - - private LLMResp llmResp; - - private QueryReq request; - - private NL2SQLTool commonAgentTool; - - private List linkingValues; -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java deleted file mode 100644 index 39a94bd8f..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.rule; - -import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentToolType; -import com.tencent.supersonic.chat.core.agent.RuleParserTool; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import java.util.List; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -@Slf4j -public class AgentCheckParser implements SemanticParser { - - @Override - public void parse(QueryContext queryContext, ChatContext chatContext) { - List queries = queryContext.getCandidateQueries(); - log.info("query size before agent filter:{}", queryContext.getCandidateQueries().size()); - filterQueries(queryContext, queries); - log.info("query size after agent filter: {}", queryContext.getCandidateQueries().size()); - } - - private void filterQueries(QueryContext queryContext, List queries) { - Agent agent = queryContext.getAgent(); - if (agent == null) { - return; - } - List queryTools = getRuleTools(agent); - if (CollectionUtils.isEmpty(queryTools)) { - queryContext.setCandidateQueries(Lists.newArrayList()); - return; - } - log.info("agent name :{}, queries resolved: {}", agent.getName(), - queries.stream().map(SemanticQuery::getQueryMode).collect(Collectors.toList())); - queries.removeIf(query -> { - for (RuleParserTool tool : queryTools) { - if (CollectionUtils.isNotEmpty(tool.getQueryModes()) - && !tool.getQueryModes().contains(query.getQueryMode())) { - return true; - } - if (CollectionUtils.isEmpty(tool.getDataSetIds())) { - return true; - } - if (tool.isContainsAllModel()) { - return false; - } - return !tool.getDataSetIds().contains(query.getParseInfo().getDataSetId()); - } - return true; - }); - queryContext.setCandidateQueries(queries); - log.info("agent name :{}, rule queries witch can be supported by agent :{}", agent.getName(), - queries.stream().map(SemanticQuery::getQueryMode).collect(Collectors.toList())); - } - - private static List getRuleTools(Agent agent) { - if (agent == null) { - return Lists.newArrayList(); - } - List tools = agent.getTools(AgentToolType.NL2SQL_RULE); - if (CollectionUtils.isEmpty(tools)) { - return Lists.newArrayList(); - } - return tools.stream().map(tool -> JSONObject.parseObject(tool, RuleParserTool.class)) - .collect(Collectors.toList()); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/LLMSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/LLMSemanticQuery.java deleted file mode 100644 index 490d96c58..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/LLMSemanticQuery.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.tencent.supersonic.chat.core.query.llm; - -import com.tencent.supersonic.chat.core.query.BaseSemanticQuery; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public abstract class LLMSemanticQuery extends BaseSemanticQuery { -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerReq.java deleted file mode 100644 index 71efa5e66..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerReq.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.tencent.supersonic.chat.core.query.llm.analytics; - - -import lombok.Data; - -@Data -public class LLMAnswerReq { - - private String queryText; - - private String pluginOutput; - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerResp.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerResp.java deleted file mode 100644 index b8037c9e8..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/LLMAnswerResp.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.tencent.supersonic.chat.core.query.llm.analytics; - - -import lombok.Data; - -@Data -public class LLMAnswerResp { - private String assistantMessage; - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java deleted file mode 100644 index 3a58bf05c..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.tencent.supersonic.chat.core.query.llm.analytics; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -@Slf4j -@Component -public class MetricAnalyzeQuery extends LLMSemanticQuery { - - - public static final String QUERY_MODE = "METRIC_INTERPRET"; - - public MetricAnalyzeQuery() { - QueryManager.register(this); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - - @Override - public QueryResult execute(User user) throws SqlParseException { - QueryStructReq queryStructReq = convertQueryStruct(); - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - - SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructReq, user); - String text = generateTableText(semanticQueryResp); - Map properties = parseInfo.getProperties(); - Map replacedMap = new HashMap<>(); - String textReplaced = replaceText((String) properties.get("queryText"), - parseInfo.getElementMatches(), replacedMap); - String answer = replaceAnswer(fetchInterpret(textReplaced, text), replacedMap); - QueryResult queryResult = new QueryResult(); - List queryColumns = Lists.newArrayList(new QueryColumn("结果", "string", "answer")); - Map result = new HashMap<>(); - result.put("answer", answer); - List> resultList = Lists.newArrayList(); - resultList.add(result); - queryResult.setQueryResults(resultList); - queryResult.setQueryColumns(queryColumns); - queryResult.setQueryMode(getQueryMode()); - queryResult.setQueryState(QueryState.SUCCESS); - return queryResult; - } - - @Override - public void initS2Sql(SemanticSchema semanticSchema, User user) { - initS2SqlByStruct(semanticSchema); - } - - protected QueryStructReq convertQueryStruct() { - QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(parseInfo); - fillAggregator(queryStructReq, parseInfo.getMetrics()); - queryStructReq.setQueryType(QueryType.TAG); - return queryStructReq; - } - - private String replaceText(String text, List schemaElementMatches, - Map replacedMap) { - if (CollectionUtils.isEmpty(schemaElementMatches)) { - return text; - } - List valueSchemaElementMatches = schemaElementMatches.stream() - .filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) - || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); - for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) { - String detectWord = schemaElementMatch.getDetectWord(); - if (StringUtils.isBlank(detectWord)) { - continue; - } - text = text.replace(detectWord, "xxx"); - replacedMap.put("xxx", detectWord); - } - return text; - } - - private void fillAggregator(QueryStructReq queryStructReq, Set schemaElements) { - queryStructReq.getAggregators().clear(); - for (SchemaElement schemaElement : schemaElements) { - Aggregator aggregator = new Aggregator(); - aggregator.setColumn(schemaElement.getBizName()); - aggregator.setFunc(AggOperatorEnum.SUM); - aggregator.setNameCh(schemaElement.getName()); - queryStructReq.getAggregators().add(aggregator); - } - } - - private String replaceAnswer(String text, Map replacedMap) { - for (String key : replacedMap.keySet()) { - text = text.replaceAll(key, replacedMap.get(key)); - } - return text; - } - - public static String generateTableText(SemanticQueryResp result) { - StringBuilder tableBuilder = new StringBuilder(); - for (QueryColumn column : result.getColumns()) { - tableBuilder.append(column.getName()).append("\t"); - } - tableBuilder.append("\n"); - for (Map row : result.getResultList()) { - for (QueryColumn column : result.getColumns()) { - tableBuilder.append(row.get(column.getNameEn())).append("\t"); - } - tableBuilder.append("\n"); - } - return tableBuilder.toString(); - } - - public String fetchInterpret(String queryText, String dataText) { - return ""; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java deleted file mode 100644 index 1581860ba..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.tencent.supersonic.chat.core.query.llm.s2sql; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -@Slf4j -@Component -public class LLMSqlQuery extends LLMSemanticQuery { - - public static final String QUERY_MODE = "LLM_S2SQL"; - protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - - public LLMSqlQuery() { - QueryManager.register(this); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - - @Override - public QueryResult execute(User user) { - - long startTime = System.currentTimeMillis(); - String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId()); - SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); - - log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); - - QueryResult queryResult = new QueryResult(); - if (Objects.nonNull(queryResp)) { - queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); - } - String resultQql = queryResp == null ? null : queryResp.getSql(); - List> resultList = queryResp == null ? new ArrayList<>() : queryResp.getResultList(); - List columns = queryResp == null ? new ArrayList<>() : queryResp.getColumns(); - queryResult.setQuerySql(resultQql); - queryResult.setQueryResults(resultList); - queryResult.setQueryColumns(columns); - queryResult.setQueryMode(QUERY_MODE); - queryResult.setQueryState(QueryState.SUCCESS); - - parseInfo.setProperties(null); - return queryResult; - } - - @Override - public void initS2Sql(SemanticSchema semanticSchema, User user) { - SqlInfo sqlInfo = parseInfo.getSqlInfo(); - sqlInfo.setCorrectS2SQL(sqlInfo.getS2SQL()); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageQuery.java deleted file mode 100644 index b5f1d0d65..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageQuery.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.tencent.supersonic.chat.core.query.plugin.webpage; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginParseResult; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.util.JsonUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.util.Map; - -@Slf4j -@Component -public class WebPageQuery extends PluginSemanticQuery { - - public static String QUERY_MODE = "WEB_PAGE"; - - public WebPageQuery() { - QueryManager.register(this); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - - @Override - public QueryResult execute(User user) { - QueryResult queryResult = new QueryResult(); - queryResult.setQueryMode(QUERY_MODE); - Map properties = parseInfo.getProperties(); - PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)), - PluginParseResult.class); - WebPageResp webPageResponse = buildResponse(pluginParseResult); - queryResult.setResponse(webPageResponse); - queryResult.setQueryState(QueryState.SUCCESS); - return queryResult; - } - - protected WebPageResp buildResponse(PluginParseResult pluginParseResult) { - Plugin plugin = pluginParseResult.getPlugin(); - WebPageResp webPageResponse = new WebPageResp(); - webPageResponse.setName(plugin.getName()); - webPageResponse.setPluginId(plugin.getId()); - webPageResponse.setPluginType(plugin.getType()); - WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class); - WebBase webBase = fillWebBaseResult(webPage, pluginParseResult); - webPageResponse.setWebPage(webBase); - return webPageResponse; - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceResp.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceResp.java deleted file mode 100644 index 0fbab079e..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceResp.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.chat.core.query.plugin.webservice; - -import com.tencent.supersonic.chat.core.query.plugin.WebBase; -import lombok.Data; - - -@Data -public class WebServiceResp { - - private WebBase webBase; - - private Object result; - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java deleted file mode 100644 index 1983a8db9..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import org.springframework.stereotype.Component; - -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; -@Component -public class MetricModelQuery extends MetricSemanticQuery { - - public static final String QUERY_MODE = "METRIC_MODEL"; - - public MetricModelQuery() { - super(); - queryMatcher.addOption(SchemaElementType.DATASET, OPTIONAL, AT_MOST, 1); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - - @Override - public QueryResult execute(User user) { - QueryResult queryResult = super.execute(user); - return queryResult; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java deleted file mode 100644 index d34beedec..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.tencent.supersonic.chat.core.query.semantic; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -@Slf4j -public abstract class BaseSemanticInterpreter implements SemanticInterpreter { - - protected final Cache> dataSetSchemaCache = - CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); - - @SneakyThrows - public List fetchDataSetSchema(List ids, Boolean cacheEnable) { - if (cacheEnable) { - return dataSetSchemaCache.get(String.valueOf(ids), () -> { - List data = doFetchDataSetSchema(ids); - dataSetSchemaCache.put(String.valueOf(ids), data); - return data; - }); - } - return doFetchDataSetSchema(ids); - } - - @Override - public DataSetSchema getDataSetSchema(Long dataSetId, Boolean cacheEnable) { - List ids = new ArrayList<>(); - ids.add(dataSetId); - List dataSetSchemaResps = fetchDataSetSchema(ids, cacheEnable); - if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { - Optional dataSetSchemaResp = dataSetSchemaResps.stream() - .filter(d -> d.getId().equals(dataSetId)).findFirst(); - if (dataSetSchemaResp.isPresent()) { - DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get(); - return DataSetSchemaBuilder.build(dataSetSchema); - } - } - return null; - } - - @Override - public List getDataSetSchema() { - return getDataSetSchema(new ArrayList<>()); - } - - @Override - public List getDataSetSchema(List ids) { - List domainSchemaList = new ArrayList<>(); - - for (DataSetSchemaResp resp : fetchDataSetSchema(ids, true)) { - domainSchemaList.add(DataSetSchemaBuilder.build(resp)); - } - - return domainSchemaList; - } - - protected abstract List doFetchDataSetSchema(List ids); - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java deleted file mode 100644 index 64f11af19..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.tencent.supersonic.chat.core.query.semantic; - -import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; -import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.QueryService; -import com.tencent.supersonic.headless.server.service.SchemaService; -import java.util.List; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LocalSemanticInterpreter extends BaseSemanticInterpreter { - - private SchemaService schemaService; - private DimensionService dimensionService; - private MetricService metricService; - private QueryService queryService; - - @SneakyThrows - @Override - public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { - queryService = ContextUtils.getBean(QueryService.class); - if (queryStructReq.isConvertToSql()) { - return queryService.queryByReq(queryStructReq.convert(), user); - } - return queryService.queryByReq(queryStructReq, user); - } - - @Override - @SneakyThrows - public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { - queryService = ContextUtils.getBean(QueryService.class); - return queryService.queryByReq(queryMultiStructReq, user); - } - - @Override - @SneakyThrows - public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) { - queryService = ContextUtils.getBean(QueryService.class); - SemanticQueryResp object = queryService.queryByReq(querySqlReq, user); - return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class); - } - - @Override - public List doFetchDataSetSchema(List ids) { - DataSetFilterReq filter = new DataSetFilterReq(); - filter.setDataSetIds(ids); - schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.fetchDataSetSchema(filter); - } - - @Override - public List getDomainList(User user) { - schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.getDomainList(user); - } - - @Override - public List getDataSetList(Long domainId) { - schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.getDataSetList(domainId); - } - - @Override - public ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception { - queryService = ContextUtils.getBean(QueryService.class); - return queryService.explain(explainSqlReq, user); - } - - @Override - public PageInfo getDimensionPage(PageDimensionReq pageDimensionCmd) { - dimensionService = ContextUtils.getBean(DimensionService.class); - return dimensionService.queryDimension(pageDimensionCmd); - } - - @Override - public PageInfo getMetricPage(PageMetricReq pageMetricReq, User user) { - metricService = ContextUtils.getBean(MetricService.class); - return metricService.queryMetric(pageMetricReq, user); - } - - @Override - public List getDomainDataSetTree() { - return schemaService.getDomainDataSetTree(); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java deleted file mode 100644 index 02c393813..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java +++ /dev/null @@ -1,253 +0,0 @@ -package com.tencent.supersonic.chat.core.query.semantic; - -import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; - -import com.github.pagehelper.PageInfo; -import com.google.gson.Gson; -import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.config.DefaultSemanticConfig; -import com.tencent.supersonic.common.pojo.ResultData; -import com.tencent.supersonic.common.pojo.enums.ReturnCode; -import com.tencent.supersonic.common.pojo.exception.CommonException; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.common.util.S2ThreadContext; -import com.tencent.supersonic.common.util.ThreadContext; -import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import java.net.URI; -import java.net.URL; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.BeanUtils; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -@Slf4j -public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { - - private S2ThreadContext s2ThreadContext; - - private AuthenticationConfig authenticationConfig; - - private ParameterizedTypeReference> structTypeRef = - new ParameterizedTypeReference>() { - }; - - private ParameterizedTypeReference> explainTypeRef = - new ParameterizedTypeReference>() { - }; - - @Override - public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { - return queryByS2SQL(queryStructReq.convert(), user); - } - - @Override - public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - return searchByRestTemplate( - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(), - new Gson().toJson(queryMultiStructReq)); - } - - @Override - public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) { - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), - new Gson().toJson(querySqlReq)); - } - - public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - fillToken(headers); - URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); - HttpEntity entity = new HttpEntity<>(jsonReq, headers); - log.info("url:{},searchByRestTemplate:{}", url, entity.getBody()); - ResultData responseBody; - try { - RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - - ResponseEntity> responseEntity = restTemplate.exchange( - requestUrl, HttpMethod.POST, entity, structTypeRef); - responseBody = responseEntity.getBody(); - log.info("ApiResponse responseBody:{}", responseBody); - SemanticQueryResp schemaResp = new SemanticQueryResp(); - if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) { - SemanticQueryResp data = responseBody.getData(); - schemaResp.setColumns(data.getColumns()); - schemaResp.setResultList(data.getResultList()); - schemaResp.setSql(data.getSql()); - schemaResp.setQueryAuthorization(data.getQueryAuthorization()); - return schemaResp; - } - } catch (Exception e) { - throw new RuntimeException("search headless interface error,url:" + url, e); - } - throw new CommonException(responseBody.getCode(), responseBody.getMsg()); - } - - @Override - public List getDomainList(User user) { - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - Object domainDescListObject = fetchHttpResult( - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainListPath(), - null, HttpMethod.GET); - return JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class); - } - - @Override - public ExplainResp explain(ExplainSqlReq explainResp, User user) throws Exception { - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - String semanticUrl = defaultSemanticConfig.getSemanticUrl(); - String explainPath = defaultSemanticConfig.getExplainPath(); - URL url = new URL(new URL(semanticUrl), explainPath); - return explain(url.toString(), JsonUtil.toString(explainResp)); - } - - public ExplainResp explain(String url, String jsonReq) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - fillToken(headers); - URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); - HttpEntity entity = new HttpEntity<>(jsonReq, headers); - log.info("url:{},explain:{}", url, entity.getBody()); - ResultData responseBody; - try { - RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - - ResponseEntity> responseEntity = restTemplate.exchange( - requestUrl, HttpMethod.POST, entity, explainTypeRef); - log.info("ApiResponse responseBody:{}", responseEntity); - responseBody = responseEntity.getBody(); - if (Objects.nonNull(responseBody.getData())) { - return responseBody.getData(); - } - return null; - } catch (Exception e) { - throw new RuntimeException("explain interface error,url:" + url, e); - } - } - - public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - fillToken(headers); - URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); - ParameterizedTypeReference> responseTypeRef = - new ParameterizedTypeReference>() { - }; - HttpEntity entity = new HttpEntity<>(JsonUtil.toString(bodyJson), headers); - try { - RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - ResponseEntity> responseEntity = restTemplate.exchange(requestUrl, - httpMethod, entity, responseTypeRef); - ResultData responseBody = responseEntity.getBody(); - log.debug("ApiResponse responseBody:{}", responseBody); - if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) { - Object data = responseBody.getData(); - return data; - } - } catch (Exception e) { - throw new RuntimeException("fetchModelSchema interface error", e); - } - throw new RuntimeException("fetchModelSchema interface error"); - } - - public void fillToken(HttpHeaders headers) { - s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class); - authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class); - ThreadContext threadContext = s2ThreadContext.get(); - if (Objects.nonNull(threadContext) && Strings.isNotEmpty(threadContext.getToken())) { - if (Objects.nonNull(authenticationConfig) && Strings.isNotEmpty( - authenticationConfig.getTokenHttpHeaderKey())) { - headers.set(authenticationConfig.getTokenHttpHeaderKey(), threadContext.getToken()); - } - } else { - log.debug("threadContext is null:{}", Objects.isNull(threadContext)); - } - } - - @Override - public PageInfo getMetricPage(PageMetricReq pageMetricCmd, User user) { - String body = JsonUtil.toString(pageMetricCmd); - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - log.info("url:{}", defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath()); - Object dimensionListObject = fetchHttpResult( - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath(), - body, HttpMethod.POST); - LinkedHashMap map = (LinkedHashMap) dimensionListObject; - PageInfo metricDescObjectPageInfo = generatePageInfo(map); - PageInfo metricDescPageInfo = new PageInfo<>(); - BeanUtils.copyProperties(metricDescObjectPageInfo, metricDescPageInfo); - metricDescPageInfo.setList(metricDescPageInfo.getList()); - return metricDescPageInfo; - } - - @Override - public PageInfo getDimensionPage(PageDimensionReq pageDimensionCmd) { - String body = JsonUtil.toString(pageDimensionCmd); - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - Object dimensionListObject = fetchHttpResult( - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDimensionPagePath(), - body, HttpMethod.POST); - LinkedHashMap map = (LinkedHashMap) dimensionListObject; - PageInfo dimensionDescObjectPageInfo = generatePageInfo(map); - PageInfo dimensionDescPageInfo = new PageInfo<>(); - BeanUtils.copyProperties(dimensionDescObjectPageInfo, dimensionDescPageInfo); - dimensionDescPageInfo.setList(dimensionDescPageInfo.getList()); - return dimensionDescPageInfo; - } - - private PageInfo generatePageInfo(LinkedHashMap map) { - PageInfo pageInfo = new PageInfo<>(); - pageInfo.setList((List) map.get(LIST_LOWER)); - Integer total = (Integer) map.get(TOTAL_LOWER); - pageInfo.setTotal(total); - Integer pageSize = (Integer) map.get(PAGESIZE_LOWER); - pageInfo.setPageSize(pageSize); - pageInfo.setPages((int) Math.ceil((double) total / pageSize)); - return pageInfo; - } - - @Override - protected List doFetchDataSetSchema(List ids) { - return null; - } - - @Override - public List getDomainDataSetTree() { - return null; - } - - @Override - public List getDataSetList(Long domainId) { - return null; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java deleted file mode 100644 index eaac77815..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.tencent.supersonic.chat.core.query.semantic; - -import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; - -import java.util.List; - -/** - * A semantic layer provides a simplified and consistent dataSet of data from multiple sources. - * It abstracts away the complexity of the underlying data sources and provides a unified dataSet - * of the data that is easier to understand and use. - *

- * The interface defines methods for getting metadata as well as querying data in the semantic layer. - * Implementations of this interface should provide concrete implementations that interact with the - * underlying data sources and return results in a consistent format. Or it can be implemented - * as proxy to a remote semantic service. - *

- */ -public interface SemanticInterpreter { - - SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user); - - SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); - - SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); - - List getDataSetSchema(); - - List getDataSetSchema(List ids); - - DataSetSchema getDataSetSchema(Long model, Boolean cacheEnable); - - PageInfo getDimensionPage(PageDimensionReq pageDimensionReq); - - PageInfo getMetricPage(PageMetricReq pageDimensionReq, User user); - - List getDomainList(User user); - - ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; - - List fetchDataSetSchema(List ids, Boolean cacheEnable); - - List getDataSetList(Long domainId); - - List getDomainDataSetTree(); - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java deleted file mode 100644 index e3b3e842e..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.tencent.supersonic.chat.core.utils; - -import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; -import com.tencent.supersonic.chat.core.parser.LLMProxy; -import com.tencent.supersonic.chat.core.parser.sql.llm.DataSetResolver; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.common.util.ContextUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.core.io.support.SpringFactoriesLoader; - -import java.util.Map; -import java.util.Objects; - -@Slf4j -public class ComponentFactory { - - private static SemanticInterpreter semanticInterpreter; - private static LLMProxy llmProxy; - private static DataSetResolver modelResolver; - - public static SemanticInterpreter getSemanticLayer() { - if (Objects.isNull(semanticInterpreter)) { - semanticInterpreter = init(SemanticInterpreter.class); - } - return semanticInterpreter; - } - - public static LLMProxy getLLMProxy() { - //1.Preferentially retrieve from environment variables - String llmProxyEnv = System.getenv("llmProxy"); - if (StringUtils.isNotBlank(llmProxyEnv)) { - Map implementations = ContextUtils.getBeansOfType(LLMProxy.class); - llmProxy = implementations.entrySet().stream() - .filter(entry -> entry.getKey().equalsIgnoreCase(llmProxyEnv)) - .map(Map.Entry::getValue) - .findFirst() - .orElse(null); - } - //2.default JavaLLMProxy - if (Objects.isNull(llmProxy)) { - llmProxy = ContextUtils.getBean(JavaLLMProxy.class); - } - log.info("llmProxy:{}", llmProxy); - return llmProxy; - } - - public static DataSetResolver getModelResolver() { - if (Objects.isNull(modelResolver)) { - modelResolver = init(DataSetResolver.class); - } - return modelResolver; - } - - private static T init(Class factoryType) { - return SpringFactoriesLoader.loadFactories(factoryType, - Thread.currentThread().getContextClassLoader()).get(0); - } -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java deleted file mode 100644 index 892061ed3..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java +++ /dev/null @@ -1,158 +0,0 @@ -package com.tencent.supersonic.chat.core.utils; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; -import com.tencent.supersonic.common.config.EmbeddingConfig; -import com.tencent.supersonic.common.util.ComponentFactory; -import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; -import com.tencent.supersonic.common.util.embedding.Retrieval; -import com.tencent.supersonic.common.util.embedding.RetrieveQuery; -import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; -import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.net.URI; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -@Slf4j -@Component -public class SimilarQueryManager { - - private EmbeddingConfig embeddingConfig; - - private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); - - - public SimilarQueryManager(EmbeddingConfig embeddingConfig) { - this.embeddingConfig = embeddingConfig; - } - - public void saveSimilarQuery(SimilarQueryReq similarQueryReq) { - if (StringUtils.isBlank(embeddingConfig.getUrl())) { - return; - } - String queryText = similarQueryReq.getQueryText(); - try { - String uniqueId = generateUniqueId(similarQueryReq.getQueryId(), similarQueryReq.getParseId()); - EmbeddingQuery embeddingQuery = new EmbeddingQuery(); - embeddingQuery.setQueryId(uniqueId); - embeddingQuery.setQuery(queryText); - - Map metaData = new HashMap<>(); - metaData.put("modelId", similarQueryReq.getDataSetId()); - metaData.put("agentId", similarQueryReq.getAgentId()); - embeddingQuery.setMetadata(metaData); - String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); - s2EmbeddingStore.addQuery(solvedQueryCollection, Lists.newArrayList(embeddingQuery)); - } catch (Exception e) { - log.warn("save history question to embedding failed, queryText:{}", queryText, e); - } - } - - public List recallSimilarQuery(String queryText, Integer agentId) { - if (StringUtils.isBlank(embeddingConfig.getUrl())) { - return Lists.newArrayList(); - } - List similarQueryRecallResps = Lists.newArrayList(); - try { - String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); - int solvedQueryResultNum = embeddingConfig.getSolvedQueryResultNum(); - - Map filterCondition = new HashMap<>(); - filterCondition.put("agentId", String.valueOf(agentId)); - RetrieveQuery retrieveQuery = RetrieveQuery.builder() - .queryTextsList(Lists.newArrayList(queryText)) - .filterCondition(filterCondition) - .build(); - List resultList = s2EmbeddingStore.retrieveQuery(solvedQueryCollection, retrieveQuery, - solvedQueryResultNum); - - log.info("[embedding] recognize result body:{}", resultList); - Set querySet = new HashSet<>(); - if (CollectionUtils.isNotEmpty(resultList)) { - for (RetrieveQueryResult retrieveQueryResult : resultList) { - List retrievals = retrieveQueryResult.getRetrieval(); - for (Retrieval retrieval : retrievals) { - if (queryText.equalsIgnoreCase(retrieval.getQuery())) { - continue; - } - if (querySet.contains(retrieval.getQuery())) { - continue; - } - String id = retrieval.getId(); - SimilarQueryRecallResp similarQueryRecallResp = SimilarQueryRecallResp.builder() - .queryText(retrieval.getQuery()) - .queryId(getQueryId(id)).parseId(getParseId(id)) - .build(); - similarQueryRecallResps.add(similarQueryRecallResp); - querySet.add(retrieval.getQuery()); - } - } - } - - } catch (Exception e) { - log.warn("recall similar solved query failed, queryText:{}", queryText); - } - return similarQueryRecallResps; - } - - private String generateUniqueId(Long queryId, Integer parseId) { - String uniqueId = queryId + String.valueOf(parseId); - if (parseId < 10) { - uniqueId = queryId + String.format("0%s", parseId); - } - return uniqueId; - } - - private Long getQueryId(String uniqueId) { - return Long.parseLong(uniqueId) / 100; - } - - private Integer getParseId(String uniqueId) { - return Integer.parseInt(uniqueId) % 100; - } - - private ResponseEntity doRequest(String path, String jsonBody) { - if (Strings.isEmpty(embeddingConfig.getUrl())) { - return ResponseEntity.of(Optional.empty()); - } - String url = embeddingConfig.getUrl() + path; - try { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - headers.setLocation(URI.create(url)); - URI requestUrl = UriComponentsBuilder - .fromHttpUrl(url).build().encode().toUri(); - HttpEntity entity = new HttpEntity<>(jsonBody, headers); - log.info("[embedding] request body :{}, url:{}", jsonBody, url); - RestTemplate restTemplate = new RestTemplate(); - ResponseEntity responseEntity = restTemplate.exchange(requestUrl, - HttpMethod.POST, entity, new ParameterizedTypeReference() { - }); - log.info("[embedding] result body:{}", responseEntity); - return responseEntity; - } catch (Exception e) { - log.warn("connect to embedding service failed, url:{}", url); - } - return ResponseEntity.of(Optional.empty()); - } - -} diff --git a/chat/pom.xml b/chat/pom.xml index 46eae4dee..c4704c08b 100644 --- a/chat/pom.xml +++ b/chat/pom.xml @@ -6,7 +6,6 @@ pom api - core server diff --git a/chat/server/pom.xml b/chat/server/pom.xml index 2e7062e18..6ea7874bf 100644 --- a/chat/server/pom.xml +++ b/chat/server/pom.xml @@ -11,19 +11,10 @@ chat-server - - org.springframework - spring-context - org.projectlombok lombok - - com.tencent.supersonic - common - ${project.version} - com.tencent.supersonic auth-api @@ -36,13 +27,7 @@ com.tencent.supersonic - headless-core - ${project.version} - compile - - - com.tencent.supersonic - chat-core + headless-server ${project.version} compile @@ -51,12 +36,6 @@ junit test - - org.mockito - mockito-inline - ${mockito-inline.version} - test - \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java similarity index 90% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java index f32e3a460..0b81f4057 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; import org.springframework.util.CollectionUtils; @@ -66,7 +67,11 @@ public class Agent extends RecordInfo { } public Set getDataSetIds() { - return getDataSetIds(null); + Set dataSetIds = getDataSetIds(null); + if (containsAllModel(dataSetIds)) { + return Sets.newHashSet(); + } + return dataSetIds; } public Set getDataSetIds(AgentToolType agentToolType) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentConfig.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java index b736888dc..eb460d56d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import com.google.common.collect.Lists; import lombok.AllArgsConstructor; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentTool.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentTool.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentTool.java index e1918a6e3..16daab2cf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentTool.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentToolType.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentToolType.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java index 4b4244f7f..8a74be690 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/AgentToolType.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/AgentToolType.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import java.util.HashMap; import java.util.Map; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/LLMParserTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java similarity index 74% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/LLMParserTool.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java index 686c5a0cc..a582f4475 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/LLMParserTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/LLMParserTool.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java index 4758b4422..23df168d2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/NL2SQLTool.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import lombok.AllArgsConstructor; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/PluginTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java similarity index 72% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/PluginTool.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java index 6dfd73a44..485332eff 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/PluginTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/PluginTool.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java index 687e9fece..37dc7d4f7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/RuleParserTool.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.agent; +package com.tencent.supersonic.chat.server.agent; import lombok.Data; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java index 2818e57e3..21f63dcc6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/DimValueDO.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; -import com.tencent.supersonic.chat.core.config.DefaultMetric; -import com.tencent.supersonic.chat.core.config.Dim4Dict; -import java.util.ArrayList; -import java.util.List; - +import com.tencent.supersonic.headless.core.config.DefaultMetric; +import com.tencent.supersonic.headless.core.config.Dim4Dict; import lombok.Data; import lombok.ToString; +import java.util.ArrayList; +import java.util.List; + @Data @ToString diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java index 58c6beb8d..6417194b8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatContextRepository.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.chat.server.persistence.repository; - -import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.ChatContext; public interface ChatContextRepository { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java index 271075257..dda1aea8b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/ChatQueryRepository.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.chat.server.persistence.repository; import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; + import java.util.List; public interface ChatQueryRepository { @@ -25,8 +25,8 @@ public interface ChatQueryRepository { int updateChatQuery(ChatQueryDO chatQueryDO); - List batchSaveParseInfo(ChatContext chatCtx, QueryContext queryContext, - ParseResp parseResult, List candidateParses); + List batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult, + List candidateParses); ChatParseDO getParseInfo(Long questionId, int parseId); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/StatisticsRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/StatisticsRepository.java deleted file mode 100644 index ea4431c45..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/StatisticsRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.repository; - - -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; - -import java.util.List; - -public interface StatisticsRepository { - - void batchSaveStatistics(List list); -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java index 240c58bf2..77eccb1a2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -1,16 +1,9 @@ package com.tencent.supersonic.chat.server.persistence.repository.impl; -import com.alibaba.fastjson.JSONObject; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDOExample; @@ -21,11 +14,10 @@ import com.tencent.supersonic.chat.server.persistence.mapper.custom.ShowCaseCust import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.PageUtils; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -33,6 +25,12 @@ import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + @Repository @Primary @Slf4j @@ -108,21 +106,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { queryResult.setQueryId(chatQueryDO.getQuestionId()); queryResp.setQueryResult(queryResult); } - if (StringUtils.isNotBlank(chatQueryDO.getSimilarQueries())) { - List similarQueries = JSONObject.parseArray(chatQueryDO.getSimilarQueries(), - SimilarQueryRecallResp.class); - queryResp.setSimilarQueries(similarQueries); - } return queryResp; } - public Long createChatQuery(ParseResp parseResult, ChatContext chatCtx, QueryContext queryContext) { + public Long createChatQuery(ParseResp parseResult, ChatParseReq chatParseReq) { ChatQueryDO chatQueryDO = new ChatQueryDO(); - chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId())); + chatQueryDO.setChatId(Long.valueOf(chatParseReq.getChatId())); chatQueryDO.setCreateTime(new java.util.Date()); - chatQueryDO.setUserName(queryContext.getUser().getName()); - chatQueryDO.setQueryText(queryContext.getQueryText()); - chatQueryDO.setAgentId(queryContext.getAgentId()); + chatQueryDO.setUserName(chatParseReq.getUser().getName()); + chatQueryDO.setQueryText(chatParseReq.getQueryText()); + chatQueryDO.setAgentId(chatParseReq.getAgentId()); chatQueryDO.setQueryResult(""); try { chatQueryDOMapper.insert(chatQueryDO); @@ -135,24 +128,24 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } @Override - public List batchSaveParseInfo(ChatContext chatCtx, QueryContext queryContext, + public List batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult, List candidateParses) { - Long queryId = createChatQuery(parseResult, chatCtx, queryContext); + Long queryId = createChatQuery(parseResult, chatParseReq); List chatParseDOList = new ArrayList<>(); - getChatParseDO(chatCtx, queryContext, queryId, candidateParses, chatParseDOList); + getChatParseDO(chatParseReq, queryId, candidateParses, chatParseDOList); if (!CollectionUtils.isEmpty(candidateParses)) { chatParseMapper.batchSaveParseInfo(chatParseDOList); } return chatParseDOList; } - public void getChatParseDO(ChatContext chatCtx, QueryContext queryContext, Long queryId, + public void getChatParseDO(ChatParseReq chatParseReq, Long queryId, List parses, List chatParseDOList) { for (int i = 0; i < parses.size(); i++) { ChatParseDO chatParseDO = new ChatParseDO(); - chatParseDO.setChatId(Long.valueOf(chatCtx.getChatId())); + chatParseDO.setChatId(Long.valueOf(chatParseReq.getChatId())); chatParseDO.setQuestionId(queryId); - chatParseDO.setQueryText(queryContext.getQueryText()); + chatParseDO.setQueryText(chatParseReq.getQueryText()); chatParseDO.setParseInfo(JsonUtil.toString(parses.get(i))); chatParseDO.setIsCandidate(1); if (i == 0) { @@ -160,7 +153,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } chatParseDO.setParseId(parses.get(i).getId()); chatParseDO.setCreateTime(new java.util.Date()); - chatParseDO.setUserName(queryContext.getUser().getName()); + chatParseDO.setUserName(chatParseReq.getUser().getName()); chatParseDOList.add(chatParseDO); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/StatisticsRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/StatisticsRepositoryImpl.java deleted file mode 100644 index 8bbcc742a..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/StatisticsRepositoryImpl.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.chat.server.persistence.repository.impl; - -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; -import com.tencent.supersonic.chat.server.persistence.mapper.StatisticsMapper; -import com.tencent.supersonic.chat.server.persistence.repository.StatisticsRepository; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Repository; - -import java.util.List; - -@Repository -@Primary -@Slf4j -public class StatisticsRepositoryImpl implements StatisticsRepository { - - private final StatisticsMapper statisticsMapper; - - public StatisticsRepositoryImpl(StatisticsMapper statisticsMapper) { - this.statisticsMapper = statisticsMapper; - } - - public void batchSaveStatistics(List list) { - statisticsMapper.batchSaveStatistics(list); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/ParseMode.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java similarity index 55% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/ParseMode.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java index e5b339cc4..628cf4904 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/ParseMode.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/ParseMode.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.plugin; +package com.tencent.supersonic.chat.server.plugin; public enum ParseMode { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/Plugin.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/Plugin.java index 06fc40579..85f1234fe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/Plugin.java @@ -1,9 +1,8 @@ -package com.tencent.supersonic.chat.core.plugin; +package com.tencent.supersonic.chat.server.plugin; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.parser.plugin.ParseMode; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; import org.apache.commons.collections.CollectionUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java index 576da5196..de6ed99aa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginManager.java @@ -1,31 +1,36 @@ -package com.tencent.supersonic.chat.core.plugin; +package com.tencent.supersonic.chat.server.plugin; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentToolType; -import com.tencent.supersonic.chat.core.agent.PluginTool; -import com.tencent.supersonic.chat.core.plugin.event.PluginAddEvent; -import com.tencent.supersonic.chat.core.plugin.event.PluginDelEvent; -import com.tencent.supersonic.chat.core.plugin.event.PluginUpdateEvent; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.plugin.ParamOption; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.agent.Agent; +import com.tencent.supersonic.chat.server.agent.AgentToolType; +import com.tencent.supersonic.chat.server.agent.PluginTool; +import com.tencent.supersonic.chat.server.plugin.build.ParamOption; +import com.tencent.supersonic.chat.server.plugin.build.WebBase; +import com.tencent.supersonic.chat.server.plugin.event.PluginAddEvent; +import com.tencent.supersonic.chat.server.plugin.event.PluginDelEvent; +import com.tencent.supersonic.chat.server.plugin.event.PluginUpdateEvent; +import com.tencent.supersonic.chat.server.service.AgentService; +import com.tencent.supersonic.chat.server.service.PluginService; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.ComponentFactory; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; @@ -43,21 +48,18 @@ import java.util.stream.Collectors; @Component public class PluginManager { + @Autowired private EmbeddingConfig embeddingConfig; private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); - public PluginManager(EmbeddingConfig embeddingConfig) { - this.embeddingConfig = embeddingConfig; - } + public static List getPluginAgentCanSupport(ChatParseReq chatParseReq) { + PluginService pluginService = ContextUtils.getBean(PluginService.class); + AgentService agentService = ContextUtils.getBean(AgentService.class); + Agent agent = agentService.getAgent(chatParseReq.getAgentId()); - public static List getPluginAgentCanSupport(QueryContext queryContext) { - List plugins = queryContext.getPluginList(); - if (Objects.isNull(queryContext.getAgent())) { - return plugins; - } - Agent agent = queryContext.getAgent(); - if (agent == null) { + List plugins = pluginService.getPluginList(); + if (Objects.isNull(agent)) { return plugins; } List pluginIds = getPluginTools(agent).stream().map(PluginTool::getPlugins) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java similarity index 71% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseConfig.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java index c02a6dde6..ace38a4ba 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseConfig.java @@ -1,15 +1,14 @@ -package com.tencent.supersonic.chat.core.plugin; +package com.tencent.supersonic.chat.server.plugin; -import com.tencent.supersonic.chat.core.parser.plugin.function.Parameters; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; import java.io.Serializable; import java.util.List; -import lombok.NoArgsConstructor; -import lombok.ToString; @Data @Builder @@ -18,8 +17,6 @@ import lombok.ToString; @NoArgsConstructor public class PluginParseConfig implements Serializable { - public Parameters parameters; - public List examples; private String name; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseResult.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseResult.java similarity index 60% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseResult.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseResult.java index 81bc467c8..8cb7b2170 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginParseResult.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginParseResult.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.core.plugin; +package com.tencent.supersonic.chat.server.plugin; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import lombok.Data; @Data diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java index 91b257977..f44402d80 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/PluginRecallResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.plugin; +package com.tencent.supersonic.chat.server.plugin; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/ParamOption.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java similarity index 90% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/ParamOption.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java index ea1b7f386..d85f3331b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/ParamOption.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/ParamOption.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.query.plugin; +package com.tencent.supersonic.chat.server.plugin.build; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java index 17df377e6..f23647e7b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/PluginSemanticQuery.java @@ -1,16 +1,17 @@ -package com.tencent.supersonic.chat.core.query.plugin; +package com.tencent.supersonic.chat.server.plugin.build; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.server.plugin.PluginParseResult; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.plugin.PluginParseResult; -import com.tencent.supersonic.chat.core.query.BaseSemanticQuery; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.core.chat.query.BaseSemanticQuery; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,11 +19,6 @@ import java.util.Map; @Slf4j public abstract class PluginSemanticQuery extends BaseSemanticQuery { - @Override - public String explain(User user) { - return null; - } - @Override public void initS2Sql(SemanticSchema semanticSchema, User user) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/WebBase.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java similarity index 83% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/WebBase.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java index 98cec1529..86e65201d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/WebBase.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/WebBase.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.core.query.plugin; +package com.tencent.supersonic.chat.server.plugin.build; import com.google.common.collect.Lists; import lombok.Data; + import java.util.List; @Data diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java new file mode 100644 index 000000000..12cabed9d --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageQuery.java @@ -0,0 +1,46 @@ +package com.tencent.supersonic.chat.server.plugin.build.webpage; + +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginParseResult; +import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery; +import com.tencent.supersonic.chat.server.plugin.build.WebBase; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.parser.SqlParseException; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class WebPageQuery extends PluginSemanticQuery { + + public static String QUERY_MODE = "WEB_PAGE"; + + public WebPageQuery() { + QueryManager.register(this); + } + + @Override + public String getQueryMode() { + return QUERY_MODE; + } + + @Override + public SemanticQueryReq buildSemanticQueryReq() throws SqlParseException { + return null; + } + + protected WebPageResp buildResponse(PluginParseResult pluginParseResult) { + Plugin plugin = pluginParseResult.getPlugin(); + WebPageResp webPageResponse = new WebPageResp(); + webPageResponse.setName(plugin.getName()); + webPageResponse.setPluginId(plugin.getId()); + webPageResponse.setPluginType(plugin.getType()); + WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class); + WebBase webBase = fillWebBaseResult(webPage, pluginParseResult); + webPageResponse.setWebPage(webBase); + return webPageResponse; + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageResp.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java index 5084c536c..67db6508d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webpage/WebPageResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webpage/WebPageResp.java @@ -1,9 +1,12 @@ -package com.tencent.supersonic.chat.core.query.plugin.webpage; +package com.tencent.supersonic.chat.server.plugin.build.webpage; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; + +import com.tencent.supersonic.chat.server.plugin.build.WebBase; import lombok.Data; + import java.util.List; + @Data public class WebPageResp { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceQuery.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java similarity index 55% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceQuery.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java index 7ebde63e9..cb479bd5b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/webservice/WebServiceQuery.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceQuery.java @@ -1,19 +1,15 @@ -package com.tencent.supersonic.chat.core.query.plugin.webservice; +package com.tencent.supersonic.chat.server.plugin.build.webservice; import com.alibaba.fastjson.JSON; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginParseResult; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.plugin.ParamOption; -import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginParseResult; +import com.tencent.supersonic.chat.server.plugin.build.ParamOption; +import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery; +import com.tencent.supersonic.chat.server.plugin.build.WebBase; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.parser.SqlParseException; import org.springframework.http.HttpEntity; @@ -24,6 +20,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; + import java.net.URI; import java.util.HashMap; import java.util.List; @@ -48,26 +45,8 @@ public class WebServiceQuery extends PluginSemanticQuery { } @Override - public QueryResult execute(User user) throws SqlParseException { - QueryResult queryResult = new QueryResult(); - queryResult.setQueryMode(QUERY_MODE); - Map properties = parseInfo.getProperties(); - PluginParseResult pluginParseResult = JsonUtil.toObject( - JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class); - WebServiceResp webServiceResponse = buildResponse(pluginParseResult); - Object object = webServiceResponse.getResult(); - // in order to show webServiceQuery result int frontend conveniently, - // webServiceResponse result format is consistent with queryByStruct result. - log.info("webServiceResponse result:{}", JsonUtil.toString(object)); - try { - Map data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class); - queryResult.setQueryResults((List>) data.get("resultList")); - queryResult.setQueryColumns((List) data.get("columns")); - queryResult.setQueryState(QueryState.SUCCESS); - } catch (Exception e) { - log.info("webServiceResponse result has an exception:{}", e.getMessage()); - } - return queryResult; + public SemanticQueryReq buildSemanticQueryReq() throws SqlParseException { + return null; } protected WebServiceResp buildResponse(PluginParseResult pluginParseResult) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java new file mode 100644 index 000000000..02744324e --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/build/webservice/WebServiceResp.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.chat.server.plugin.build.webservice; + + +import com.tencent.supersonic.chat.server.plugin.build.WebBase; +import lombok.Data; + + +@Data +public class WebServiceResp { + + private WebBase webBase; + + private Object result; + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginAddEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginAddEvent.java similarity index 73% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginAddEvent.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginAddEvent.java index 7c7b808cb..41d493994 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginAddEvent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginAddEvent.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.core.plugin.event; +package com.tencent.supersonic.chat.server.plugin.event; -import com.tencent.supersonic.chat.core.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.Plugin; import org.springframework.context.ApplicationEvent; public class PluginAddEvent extends ApplicationEvent { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginDelEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java similarity index 73% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginDelEvent.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java index 849a06a71..fb16cb7a4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginDelEvent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginDelEvent.java @@ -1,6 +1,7 @@ -package com.tencent.supersonic.chat.core.plugin.event; +package com.tencent.supersonic.chat.server.plugin.event; -import com.tencent.supersonic.chat.core.plugin.Plugin; + +import com.tencent.supersonic.chat.server.plugin.Plugin; import org.springframework.context.ApplicationEvent; public class PluginDelEvent extends ApplicationEvent { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginUpdateEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java similarity index 81% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginUpdateEvent.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java index cfde78975..dda1174ee 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/event/PluginUpdateEvent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/event/PluginUpdateEvent.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.core.plugin.event; +package com.tencent.supersonic.chat.server.plugin.event; -import com.tencent.supersonic.chat.core.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.Plugin; import org.springframework.context.ApplicationEvent; public class PluginUpdateEvent extends ApplicationEvent { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/PluginParser.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/PluginParser.java index c8abda334..aa0e50ed2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/PluginParser.java @@ -1,77 +1,68 @@ -package com.tencent.supersonic.chat.core.parser.plugin; +package com.tencent.supersonic.chat.server.plugin.recall; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginManager; -import com.tencent.supersonic.chat.core.plugin.PluginParseResult; -import com.tencent.supersonic.chat.core.plugin.PluginRecallResult; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginManager; +import com.tencent.supersonic.chat.server.plugin.PluginParseResult; +import com.tencent.supersonic.chat.server.plugin.PluginRecallResult; +import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import org.springframework.util.CollectionUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; - /** * PluginParser defines the basic process and common methods for recalling plugins. */ -public abstract class PluginParser implements SemanticParser { +public abstract class PluginParser { - @Override - public void parse(QueryContext queryContext, ChatContext chatContext) { - for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) { - if (queryContext.getQueryText().length() <= semanticQuery.getParseInfo().getScore() - && (QueryManager.getPluginQueryModes().contains(semanticQuery.getQueryMode()))) { - return; - } - } - if (!checkPreCondition(queryContext)) { + public void parse(ChatParseReq chatParseReq) { + if (!checkPreCondition(chatParseReq)) { return; } - PluginRecallResult pluginRecallResult = recallPlugin(queryContext); + PluginRecallResult pluginRecallResult = recallPlugin(chatParseReq); if (pluginRecallResult == null) { return; } - buildQuery(queryContext, pluginRecallResult); + buildQuery(chatParseReq, pluginRecallResult); } - public abstract boolean checkPreCondition(QueryContext queryContext); + public abstract boolean checkPreCondition(ChatParseReq chatParseReq); - public abstract PluginRecallResult recallPlugin(QueryContext queryContext); + public abstract PluginRecallResult recallPlugin(ChatParseReq chatParseReq); - public void buildQuery(QueryContext queryContext, PluginRecallResult pluginRecallResult) { + public void buildQuery(ChatParseReq chatParseReq, PluginRecallResult pluginRecallResult) { Plugin plugin = pluginRecallResult.getPlugin(); Set dataSetIds = pluginRecallResult.getDataSetIds(); if (plugin.isContainsAllModel()) { dataSetIds = Sets.newHashSet(-1L); } for (Long dataSetId : dataSetIds) { - PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); + //todo + PluginSemanticQuery pluginQuery = null; SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(dataSetId, plugin, - queryContext, pluginRecallResult.getDistance()); + null, pluginRecallResult.getDistance()); semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); semanticParseInfo.setScore(pluginRecallResult.getScore()); pluginQuery.setParseInfo(semanticParseInfo); - queryContext.getCandidateQueries().add(pluginQuery); + //chatParseReq.getCandidateQueries().add(pluginQuery); } } - protected List getPluginList(QueryContext queryContext) { - return PluginManager.getPluginAgentCanSupport(queryContext); + protected List getPluginList(ChatParseReq chatParseReq) { + return PluginManager.getPluginAgentCanSupport(chatParseReq); } protected SemanticParseInfo buildSemanticParseInfo(Long dataSetId, Plugin plugin, diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/EmbeddingRecallParser.java similarity index 75% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/EmbeddingRecallParser.java index b7153d6cd..ab3b045eb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/EmbeddingRecallParser.java @@ -1,18 +1,18 @@ -package com.tencent.supersonic.chat.core.parser.plugin.embedding; +package com.tencent.supersonic.chat.server.plugin.recall.embedding; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginManager; -import com.tencent.supersonic.chat.core.plugin.PluginRecallResult; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.parser.PythonLLMProxy; -import com.tencent.supersonic.chat.core.parser.plugin.ParseMode; -import com.tencent.supersonic.chat.core.parser.plugin.PluginParser; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.plugin.ParseMode; +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginManager; +import com.tencent.supersonic.chat.server.plugin.PluginRecallResult; +import com.tencent.supersonic.chat.server.plugin.recall.PluginParser; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; +import com.tencent.supersonic.headless.core.chat.parser.PythonLLMProxy; +import com.tencent.supersonic.headless.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -30,31 +30,30 @@ import java.util.stream.Collectors; @Slf4j public class EmbeddingRecallParser extends PluginParser { - @Override - public boolean checkPreCondition(QueryContext queryContext) { + public boolean checkPreCondition(ChatParseReq chatParseReq) { EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); if (StringUtils.isBlank(embeddingConfig.getUrl()) && ComponentFactory.getLLMProxy() instanceof PythonLLMProxy) { return false; } - List plugins = getPluginList(queryContext); + List plugins = getPluginList(chatParseReq); return !CollectionUtils.isEmpty(plugins); } - @Override - public PluginRecallResult recallPlugin(QueryContext queryContext) { - String text = queryContext.getQueryText(); + public PluginRecallResult recallPlugin(ChatParseReq chatParseReq) { + String text = chatParseReq.getQueryText(); List embeddingRetrievals = embeddingRecall(text); if (CollectionUtils.isEmpty(embeddingRetrievals)) { return null; } - List plugins = getPluginList(queryContext); + List plugins = getPluginList(chatParseReq); Map pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p)); for (Retrieval embeddingRetrieval : embeddingRetrievals) { Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId())); if (plugin == null) { continue; } - Pair> pair = PluginManager.resolve(plugin, queryContext); + //todo + Pair> pair = PluginManager.resolve(plugin, null); log.info("embedding plugin resolve: {}", pair); if (pair.getLeft()) { Set dataSetList = pair.getRight(); @@ -63,7 +62,7 @@ public class EmbeddingRecallParser extends PluginParser { } plugin.setParseMode(ParseMode.EMBEDDING_RECALL); double distance = embeddingRetrieval.getDistance(); - double score = queryContext.getQueryText().length() * (1 - distance); + double score = chatParseReq.getQueryText().length() * (1 - distance); return PluginRecallResult.builder() .plugin(plugin).dataSetIds(dataSetList).score(score).distance(distance).build(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrieval.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrieval.java similarity index 75% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrieval.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrieval.java index aaf57972b..c347eeabd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrieval.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrieval.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.plugin.embedding; +package com.tencent.supersonic.chat.server.plugin.recall.embedding; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrievalResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrievalResp.java similarity index 70% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrievalResp.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrievalResp.java index 1cf812f8f..7c0de6ac5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/RecallRetrievalResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/embedding/RecallRetrievalResp.java @@ -1,8 +1,7 @@ -package com.tencent.supersonic.chat.core.parser.plugin.embedding; +package com.tencent.supersonic.chat.server.plugin.recall.embedding; import lombok.Data; - import java.util.List; @Data diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallConfig.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionCallConfig.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallConfig.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionCallConfig.java index 1358d8604..abab66945 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallConfig.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionCallConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; +package com.tencent.supersonic.chat.server.plugin.recall.function; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionFiled.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionFiled.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionFiled.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionFiled.java index 08c0c88d6..6bbe3b568 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionFiled.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionFiled.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; +package com.tencent.supersonic.chat.server.plugin.recall.function; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionPromptGenerator.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionPromptGenerator.java similarity index 56% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionPromptGenerator.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionPromptGenerator.java index 32d16a104..576e16b0f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionPromptGenerator.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionPromptGenerator.java @@ -1,12 +1,18 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; +package com.tencent.supersonic.chat.server.plugin.recall.function; -import com.tencent.supersonic.chat.core.parser.sql.llm.InputFormat; -import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; -import java.util.List; -import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tencent.supersonic.chat.server.plugin.PluginParseConfig; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.chat.parser.llm.InputFormat; +import dev.langchain4j.model.chat.ChatLanguageModel; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.List; +import java.util.stream.Collectors; + @Component @Slf4j public class FunctionPromptGenerator { @@ -41,4 +47,29 @@ public class FunctionPromptGenerator { return String.format("工具选择如下:\n\n%s\n\n【任务说明】\n%s", functionList, instruction); } + + public FunctionResp requestFunction(FunctionReq functionReq) { + + FunctionPromptGenerator promptGenerator = ContextUtils.getBean(FunctionPromptGenerator.class); + + ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); + String functionCallPrompt = promptGenerator.generateFunctionCallPrompt(functionReq.getQueryText(), + functionReq.getPluginConfigs()); + String response = chatLanguageModel.generate(functionCallPrompt); + return functionCallParse(response); + } + + public static FunctionResp functionCallParse(String llmOutput) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(llmOutput); + String selectedTool = jsonNode.get("选择工具").asText(); + FunctionResp resp = new FunctionResp(); + resp.setToolSelection(selectedTool); + return resp; + } catch (Exception e) { + log.error("", e); + } + return null; + } } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionReq.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionReq.java similarity index 59% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionReq.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionReq.java index 292ac3e9e..ca6f05973 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionReq.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionReq.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; - -import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; -import java.util.List; +package com.tencent.supersonic.chat.server.plugin.recall.function; +import com.tencent.supersonic.chat.server.plugin.PluginParseConfig; import lombok.Builder; import lombok.Data; +import java.util.List; + @Data @Builder public class FunctionReq { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionResp.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionResp.java similarity index 58% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionResp.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionResp.java index 156abbfa1..7fe0a5b59 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionResp.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/FunctionResp.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; +package com.tencent.supersonic.chat.server.plugin.recall.function; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/Parameters.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/Parameters.java similarity index 78% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/Parameters.java rename to chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/Parameters.java index e20a19870..c78c668cb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/Parameters.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recall/function/Parameters.java @@ -1,6 +1,7 @@ -package com.tencent.supersonic.chat.core.parser.plugin.function; +package com.tencent.supersonic.chat.server.plugin.recall.function; import lombok.Data; + import java.util.List; import java.util.Map; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java index 19e39704b..e53bef063 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/ResultProcessor.java @@ -5,4 +5,5 @@ package com.tencent.supersonic.chat.server.processor; */ public interface ResultProcessor { + } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index d1e312621..fd770dd5e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -3,11 +3,11 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.server.service.SemanticService; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.server.service.impl.SemanticService; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import org.springframework.util.CollectionUtils; @@ -33,9 +33,9 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { || CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) { return; } - SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); - List dimensionRecommended = getDimensions(element.getId(), element.getDataSet()); - queryResult.setRecommendedDimensions(dimensionRecommended); + //SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); + //List dimensionRecommended = getDimensions(element.getId(), element.getDataSet()); + //queryResult.setRecommendedDimensions(dimensionRecommended); } private List getDimensions(Long metricId, Long dataSetId) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java index 626f8baa9..4108df8ad 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/ExecuteResultProcessor.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.processor.execute; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.processor.ResultProcessor; /** diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java index 535e39e4d..20051b45b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -1,25 +1,8 @@ package com.tencent.supersonic.chat.server.processor.execute; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.config.AggregatorConfig; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.QueryColumn; @@ -29,8 +12,16 @@ import com.tencent.supersonic.common.pojo.enums.RatioOverType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.core.config.AggregatorConfig; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; @@ -48,8 +39,16 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; /** * Add ratio queries for metric queries. @@ -57,7 +56,7 @@ import org.springframework.util.CollectionUtils; @Slf4j public class MetricRatioProcessor implements ExecuteResultProcessor { - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); + //private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); @Override public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { @@ -68,8 +67,8 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { || !QueryType.METRIC.equals(semanticParseInfo.getQueryType())) { return; } - AggregateInfo aggregateInfo = getAggregateInfo(queryReq.getUser(), semanticParseInfo, queryResult); - queryResult.setAggregateInfo(aggregateInfo); + //AggregateInfo aggregateInfo = getAggregateInfo(queryReq.getUser(), semanticParseInfo, queryResult); + //queryResult.setAggregateInfo(aggregateInfo); } public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { @@ -133,7 +132,7 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult)); queryStructReq.setConvertToSql(false); - SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); + SemanticQueryResp queryResp = null; MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); if (Objects.isNull(queryResp) || CollectionUtils.isEmpty(queryResp.getResultList())) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index ec91b4855..ad996f2ad 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -1,21 +1,18 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.util.ComponentFactory; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; -import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.server.service.MetaEmbeddingService; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.core.knowledge.MetaEmbeddingService; import org.springframework.util.CollectionUtils; import java.util.Collections; @@ -33,8 +30,6 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { private static final int METRIC_RECOMMEND_SIZE = 5; - private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); - @Override public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { fillSimilarMetric(queryResult.getChatContext()); @@ -54,8 +49,7 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { .filterCondition(filterCondition).queryEmbeddings(null).build(); MetaEmbeddingService metaEmbeddingService = ContextUtils.getBean(MetaEmbeddingService.class); List retrieveQueryResults = - metaEmbeddingService.retrieveQuery(Lists.newArrayList(parseInfo.getDataSetId()), - retrieveQuery, METRIC_RECOMMEND_SIZE + 1); + metaEmbeddingService.retrieveQuery(retrieveQuery, METRIC_RECOMMEND_SIZE + 1, new HashMap<>()); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java deleted file mode 100644 index dbac4bf64..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.tencent.supersonic.chat.server.processor.parse; - -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.query.llm.analytics.MetricAnalyzeQuery; -import com.tencent.supersonic.chat.server.service.SemanticService; -import com.tencent.supersonic.common.util.ContextUtils; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * EntityInfoProcessor fills core attributes of an entity so that - * users get to know which entity is parsed out. - */ -public class EntityInfoProcessor implements ParseResultProcessor { - - @Override - public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { - List semanticQueries = queryContext.getCandidateQueries(); - if (CollectionUtils.isEmpty(semanticQueries)) { - return; - } - List selectedParses = semanticQueries.stream().map(SemanticQuery::getParseInfo) - .collect(Collectors.toList()); - selectedParses.forEach(parseInfo -> { - String queryMode = parseInfo.getQueryMode(); - if (QueryManager.containsPluginQuery(queryMode) - || MetricAnalyzeQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) { - return; - } - //1. set entity info - DataSetSchema dataSetSchema = - queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, queryContext.getUser()); - if (QueryManager.isTagQuery(queryMode) - || QueryManager.isMetricQuery(queryMode)) { - parseInfo.setEntityInfo(entityInfo); - } - }); - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java index 54c61edb8..ead11facf 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseResultProcessor.java @@ -1,14 +1,10 @@ package com.tencent.supersonic.chat.server.processor.parse; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.server.processor.ResultProcessor; -/** - * A ParseResultProcessor wraps things up before returning results to users in parse stage. - */ -public interface ParseResultProcessor extends ResultProcessor { +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; - void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext); +public interface ParseResultProcessor { + + void process(ParseResp parseResp, ChatParseReq chatParseReq); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java index 358ccad36..8aab0b8fd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRecommendProcessor.java @@ -3,29 +3,30 @@ package com.tencent.supersonic.chat.server.processor.parse; import com.alibaba.fastjson.JSONObject; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp; -import com.tencent.supersonic.chat.core.utils.SimilarQueryManager; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.server.processor.ResultProcessor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /** * MetricRecommendProcessor fills recommended query based on embedding similarity. */ @Slf4j -public class QueryRecommendProcessor implements ParseResultProcessor { +public class QueryRecommendProcessor implements ResultProcessor { @Override public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { @@ -35,8 +36,9 @@ public class QueryRecommendProcessor implements ParseResultProcessor { @SneakyThrows private void doProcess(ParseResp parseResp, QueryContext queryContext) { Long queryId = parseResp.getQueryId(); + //TODO List solvedQueries = getSimilarQueries(queryContext.getQueryText(), - queryContext.getAgentId()); + null); ChatQueryDO chatQueryDO = getChatQuery(queryId); chatQueryDO.setSimilarQueries(JSONObject.toJSONString(solvedQueries)); updateChatQuery(chatQueryDO); @@ -44,8 +46,8 @@ public class QueryRecommendProcessor implements ParseResultProcessor { public List getSimilarQueries(String queryText, Integer agentId) { //1. recall solved query by queryText - SimilarQueryManager solvedQueryManager = ContextUtils.getBean(SimilarQueryManager.class); - List similarQueries = solvedQueryManager.recallSimilarQuery(queryText, agentId); + //SimilarQueryManager solvedQueryManager = ContextUtils.getBean(SimilarQueryManager.class); + List similarQueries = Lists.newArrayList(); if (CollectionUtils.isEmpty(similarQueries)) { return Lists.newArrayList(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java index 30d5f2ca6..ba374343c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/AgentController.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentToolType; +import com.tencent.supersonic.chat.server.agent.Agent; +import com.tencent.supersonic.chat.server.agent.AgentToolType; import com.tencent.supersonic.chat.server.service.AgentService; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PathVariable; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java index 01d6f8a43..727b44a5e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.server.rest; -import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -8,16 +7,9 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; -import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.server.service.SchemaService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -39,8 +31,8 @@ public class ChatConfigController { @Autowired private ConfigService configService; - - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); + @Autowired + private SchemaService schemaService; @PostMapping public Long addChatConfig(@RequestBody ChatConfigBaseReq extendBaseCmd, @@ -76,40 +68,9 @@ public class ChatConfigController { return configService.getAllChatRichConfig(); } - @GetMapping("/domainList") - public List getDomainList(HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return semanticInterpreter.getDomainList(user); - } - - //Compatible with front-end - @GetMapping("/dataSetList") - public List getDataSetList() { - return semanticInterpreter.getDataSetList(null); - } - - @GetMapping("/dataSetList/{domainId}") - public List getDataSetList(@PathVariable("domainId") Long domainId) { - return semanticInterpreter.getDataSetList(domainId); - } - - @PostMapping("/dimension/page") - public PageInfo getDimension(@RequestBody PageDimensionReq pageDimensionReq) { - return semanticInterpreter.getDimensionPage(pageDimensionReq); - } - - @PostMapping("/metric/page") - public PageInfo getMetric(@RequestBody PageMetricReq pageMetricReq, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return semanticInterpreter.getMetricPage(pageMetricReq, user); - } - @GetMapping("/getDomainDataSetTree") public List getDomainDataSetTree() { - return semanticInterpreter.getDomainDataSetTree(); + return schemaService.getDomainDataSetTree(); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java index 687fbd7a1..b4d2d8e51 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatController.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.service.ChatService; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java index 7c24b5050..4737caa31 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java @@ -1,24 +1,23 @@ package com.tencent.supersonic.chat.server.rest; -import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; -import com.tencent.supersonic.chat.server.service.QueryService; -import com.tencent.supersonic.chat.server.service.SearchService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; + /** * query controller */ @@ -27,62 +26,49 @@ import org.springframework.web.bind.annotation.RestController; public class ChatQueryController { @Autowired - @Qualifier("chatQueryService") - private QueryService queryService; - - @Autowired - private SearchService searchService; + private ChatService chatService; @PostMapping("search") - public Object search(@RequestBody QueryReq queryCtx, HttpServletRequest request, + public Object search(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request, HttpServletResponse response) { - queryCtx.setUser(UserHolder.findUser(request, response)); - return searchService.search(queryCtx); + chatParseReq.setUser(UserHolder.findUser(request, response)); + return chatService.search(chatParseReq); } @PostMapping("parse") - public Object parse(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) - throws Exception { - queryCtx.setUser(UserHolder.findUser(request, response)); - return queryService.performParsing(queryCtx); + public Object parse(@RequestBody ChatParseReq chatParseReq, + HttpServletRequest request, HttpServletResponse response) throws Exception { + chatParseReq.setUser(UserHolder.findUser(request, response)); + return chatService.performParsing(chatParseReq); } @PostMapping("execute") - public Object execute(@RequestBody ExecuteQueryReq queryReq, + public Object execute(@RequestBody ChatExecuteReq chatExecuteReq, HttpServletRequest request, HttpServletResponse response) throws Exception { - queryReq.setUser(UserHolder.findUser(request, response)); - return queryService.performExecution(queryReq); + chatExecuteReq.setUser(UserHolder.findUser(request, response)); + return chatService.performExecution(chatExecuteReq); } @PostMapping("queryContext") - public Object queryContext(@RequestBody QueryReq queryCtx, HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Object queryContext(@RequestBody QueryReq queryCtx, + HttpServletRequest request, HttpServletResponse response) { queryCtx.setUser(UserHolder.findUser(request, response)); - return queryService.queryContext(queryCtx); + return chatService.queryContext(queryCtx.getChatId()); } @PostMapping("queryData") public Object queryData(@RequestBody QueryDataReq queryData, - HttpServletRequest request, HttpServletResponse response) - throws Exception { + HttpServletRequest request, HttpServletResponse response) throws Exception { queryData.setUser(UserHolder.findUser(request, response)); - return queryService.executeDirectQuery(queryData, UserHolder.findUser(request, response)); + return chatService.queryData(queryData, UserHolder.findUser(request, response)); } @PostMapping("queryDimensionValue") public Object queryDimensionValue(@RequestBody @Valid DimensionValueReq dimensionValueReq, - HttpServletRequest request, HttpServletResponse response) - throws Exception { - return queryService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response)); + HttpServletRequest request, HttpServletResponse response) throws Exception { + return chatService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response)); } - @RequestMapping("/getEntityInfo") - public Object getEntityInfo(Long queryId, Integer parseId, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return queryService.getEntityInfo(queryId, parseId, user); - } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java index e7c13c92f..d12e1db81 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/PluginController.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; -import com.tencent.supersonic.chat.core.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.Plugin; import com.tencent.supersonic.chat.server.service.PluginService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/RecommendController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/RecommendController.java deleted file mode 100644 index 4d4e61449..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/RecommendController.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.tencent.supersonic.chat.server.rest; - -import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; -import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; -import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; -import com.tencent.supersonic.chat.server.service.RecommendService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestParam; -import java.util.List; - -/** - * recommend controller - */ -@RestController -@RequestMapping({"/api/chat/", "/openapi/chat/"}) -public class RecommendController { - - @Autowired - private RecommendService recommendService; - - @GetMapping("recommend/{modelId}") - public RecommendResp recommend(@PathVariable("modelId") Long modelId, - @RequestParam(value = "limit", required = false) Long limit) { - RecommendReq recommendReq = new RecommendReq(); - recommendReq.setModelId(modelId); - return recommendService.recommend(recommendReq, limit); - } - - @GetMapping("recommend/metric/{modelId}") - public RecommendResp recommendMetricMode(@PathVariable("modelId") Long modelId, - @RequestParam(value = "metricId", required = false) Long metricId, - @RequestParam(value = "limit", required = false) Long limit) { - RecommendReq recommendReq = new RecommendReq(); - recommendReq.setModelId(modelId); - recommendReq.setMetricId(metricId); - return recommendService.recommendMetricMode(recommendReq, limit); - } - - @GetMapping("recommend/question") - public List recommendQuestion( - @RequestParam(value = "modelId", required = false) Long modelId) { - return recommendService.recommendQuestion(modelId); - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java index 9ab72deca..8f7208eb8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/AgentService.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.agent.Agent; +import com.tencent.supersonic.chat.server.agent.Agent; import java.util.List; public interface AgentService { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java index c7e74cce3..b7ce33c0b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java @@ -2,30 +2,36 @@ package com.tencent.supersonic.chat.server.service; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.response.SearchResult; + import java.util.List; public interface ChatService { - /*** - * get the model from context - * @param chatId - * @return - */ - Long getContextModel(Integer chatId); + List search(ChatParseReq chatParseReq); - ChatContext getOrCreateContext(int chatId); + ParseResp performParsing(ChatParseReq chatParseReq); - void updateContext(ChatContext chatCtx); + QueryResult performExecution(ChatExecuteReq chatExecuteReq) throws Exception; + + Object queryData(QueryDataReq queryData, User user) throws Exception; + + SemanticParseInfo queryContext(Integer chatId); + + Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception; Boolean addChat(User user, String chatName, Integer agentId); @@ -45,13 +51,13 @@ public interface ChatService { ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId); - List batchAddParse(ChatContext chatCtx, QueryContext queryContext, ParseResp parseResult); + List batchAddParse(ChatParseReq chatParseReq, ParseResp parseResult); ChatQueryDO getLastQuery(long chatId); int updateQuery(ChatQueryDO chatQueryDO); - void updateQuery(Long questionId, int parseId, QueryResult queryResult, ChatContext chatCtx); + void saveQueryResult(ChatExecuteReq chatExecuteReq, QueryResult queryResult); ChatParseDO getParseInfo(Long questionId, int parseId); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java index c9cd64efd..0bdd2361e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/PluginService.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; -import com.tencent.supersonic.chat.core.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.Plugin; import java.util.List; import java.util.Map; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java deleted file mode 100644 index fdf1e933f..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; -import org.apache.calcite.sql.parser.SqlParseException; - -/*** - * QueryService for query and search - */ -public interface QueryService { - - ParseResp performParsing(QueryReq queryReq); - - QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception; - - SemanticParseInfo queryContext(QueryReq queryReq); - - QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException; - - EntityInfo getEntityInfo(Long queryId, Integer parseId, User user); - - Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception; -} - diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SearchService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SearchService.java deleted file mode 100644 index de70ca117..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SearchService.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.SearchResult; -import java.util.List; - -/** - * search service - */ -public interface SearchService { - - List search(QueryReq queryCtx); - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/StatisticsService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/StatisticsService.java index bd42932fa..d1535eb5d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/StatisticsService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/StatisticsService.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.server.service; -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO; import java.util.List; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCost.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCost.java deleted file mode 100644 index 9cbcbe809..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCost.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import java.lang.annotation.Documented; -import java.lang.annotation.Target; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Target({ElementType.PARAMETER, ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface TimeCost { - -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCostAOP.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCostAOP.java deleted file mode 100644 index d0475ef5d..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/TimeCostAOP.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.tencent.supersonic.chat.server.service; - -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -@Aspect -public class TimeCostAOP { - - @Pointcut("@annotation(com.tencent.supersonic.chat.server.service.TimeCost)") - private void timeCostAdvicePointcut() { - - } - - @Around("timeCostAdvicePointcut()") - public Object timeCostAdvice(ProceedingJoinPoint joinPoint) throws Throwable { - log.info("begin to add time cost!"); - Long startTime = System.currentTimeMillis(); - Object object = joinPoint.proceed(); - if (object instanceof QueryResult) { - QueryResult queryResult = (QueryResult) object; - queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime); - return queryResult; - } - return object; - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index a7734dd97..a9bff8c10 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.service.impl; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.agent.Agent; +import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO; import com.tencent.supersonic.chat.server.persistence.repository.AgentRepository; import com.tencent.supersonic.chat.server.service.AgentService; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index ce6c8f222..16f3f8d17 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -1,26 +1,36 @@ package com.tencent.supersonic.chat.server.service.impl; +import com.alibaba.fastjson.JSONObject; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; -import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO; -import com.tencent.supersonic.chat.server.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.server.persistence.repository.ChatRepository; +import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.response.SearchResult; +import com.tencent.supersonic.headless.server.service.ChatQueryService; +import com.tencent.supersonic.headless.server.service.SearchService; import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Primary; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -30,51 +40,87 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -@Service("ChatService") -@Primary + @Slf4j +@Service public class ChatServiceImpl implements ChatService { - private ChatContextRepository chatContextRepository; + @Autowired private ChatRepository chatRepository; + @Autowired private ChatQueryRepository chatQueryRepository; + @Autowired + private ChatQueryService chatQueryService; + @Autowired + private AgentService agentService; + @Autowired + private SearchService searchService; - public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository, - ChatQueryRepository chatQueryRepository) { - this.chatContextRepository = chatContextRepository; - this.chatRepository = chatRepository; - this.chatQueryRepository = chatQueryRepository; + @Override + public List search(ChatParseReq chatParseReq) { + QueryReq queryReq = buildSqlQueryReq(chatParseReq); + return searchService.search(queryReq); } @Override - public Long getContextModel(Integer chatId) { - if (Objects.isNull(chatId)) { - return null; - } - ChatContext chatContext = getOrCreateContext(chatId); - if (Objects.isNull(chatContext)) { - return null; - } - SemanticParseInfo originalSemanticParse = chatContext.getParseInfo(); - if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getDataSetId())) { - return originalSemanticParse.getDataSetId(); - } - return null; + public ParseResp performParsing(ChatParseReq chatParseReq) { + QueryReq queryReq = buildSqlQueryReq(chatParseReq); + ParseResp parseResp = chatQueryService.performParsing(queryReq); + batchAddParse(chatParseReq, parseResp); + return parseResp; } @Override - public ChatContext getOrCreateContext(int chatId) { - return chatContextRepository.getOrCreateContext(chatId); + public QueryResult performExecution(ChatExecuteReq chatExecuteReq) throws Exception { + ExecuteQueryReq executeQueryReq = buildExecuteReq(chatExecuteReq); + QueryResult queryResult = chatQueryService.performExecution(executeQueryReq); + saveQueryResult(chatExecuteReq, queryResult); + return queryResult; } @Override - public void updateContext(ChatContext chatCtx) { - log.debug("save ChatContext {}", chatCtx); - chatContextRepository.updateContext(chatCtx); + public Object queryData(QueryDataReq queryData, User user) throws Exception { + return chatQueryService.executeDirectQuery(queryData, user); + } + + @Override + public SemanticParseInfo queryContext(Integer chatId) { + return chatQueryService.queryContext(chatId); + } + + @Override + public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { + return chatQueryService.queryDimensionValue(dimensionValueReq, user); + } + + private QueryReq buildSqlQueryReq(ChatParseReq chatParseReq) { + QueryReq queryReq = new QueryReq(); + BeanMapper.mapper(chatParseReq, queryReq); + if (chatParseReq.getAgentId() == null) { + return queryReq; + } + Agent agent = agentService.getAgent(chatParseReq.getAgentId()); + if (agent == null) { + return queryReq; + } + queryReq.setDataSetIds(agent.getDataSetIds()); + return queryReq; + } + + private ExecuteQueryReq buildExecuteReq(ChatExecuteReq chatExecuteReq) { + ChatParseDO chatParseDO = getParseInfo(chatExecuteReq.getQueryId(), chatExecuteReq.getParseId()); + SemanticParseInfo parseInfo = JSONObject.parseObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + return ExecuteQueryReq.builder() + .queryId(chatExecuteReq.getQueryId()) + .chatId(chatExecuteReq.getChatId()) + .queryText(chatExecuteReq.getQueryText()) + .parseInfo(parseInfo) + .saveAnswer(chatExecuteReq.isSaveAnswer()) + .user(chatExecuteReq.getUser()) + .build(); } @Override @@ -190,18 +236,18 @@ public class ChatServiceImpl implements ChatService { } @Override - public void updateQuery(Long questionId, int parseId, QueryResult queryResult, ChatContext chatCtx) { + public void saveQueryResult(ChatExecuteReq chatExecuteReq, QueryResult queryResult) { //The history record only retains the query result of the first parse - if (parseId > 1) { + if (chatExecuteReq.getParseId() > 1) { return; } ChatQueryDO chatQueryDO = new ChatQueryDO(); - chatQueryDO.setQuestionId(questionId); + chatQueryDO.setQuestionId(chatExecuteReq.getQueryId()); chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); chatQueryDO.setQueryState(1); updateQuery(chatQueryDO); - chatRepository.updateLastQuestion(chatCtx.getChatId().longValue(), - chatCtx.getQueryText(), getCurrentTime()); + chatRepository.updateLastQuestion(chatExecuteReq.getChatId().longValue(), + chatExecuteReq.getQueryText(), getCurrentTime()); } @Override @@ -210,9 +256,9 @@ public class ChatServiceImpl implements ChatService { } @Override - public List batchAddParse(ChatContext chatCtx, QueryContext queryContext, ParseResp parseResult) { + public List batchAddParse(ChatParseReq chatParseReq, ParseResp parseResult) { List candidateParses = parseResult.getSelectedParses(); - return chatQueryRepository.batchSaveParseInfo(chatCtx, queryContext, parseResult, candidateParses); + return chatQueryRepository.batchSaveParseInfo(chatParseReq, parseResult, candidateParses); } @Override diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 9f2522a81..fd40f8495 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -3,9 +3,6 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; @@ -24,23 +21,22 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp; import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo; import com.tencent.supersonic.chat.server.config.ChatConfig; -import com.tencent.supersonic.chat.server.util.ChatConfigHelper; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.server.util.VisibilityEvent; import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.chat.server.service.SemanticService; +import com.tencent.supersonic.chat.server.util.ChatConfigHelper; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.impl.SemanticService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -62,10 +58,6 @@ public class ConfigServiceImpl implements ConfigService { private final MetricService metricService; @Autowired private SemanticService semanticService; - @Autowired - private ApplicationEventPublisher applicationEventPublisher; - - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); public ConfigServiceImpl(ChatConfigRepository chatConfigRepository, @@ -83,9 +75,7 @@ public class ConfigServiceImpl implements ConfigService { log.info("[create model extend] object:{}", JsonUtil.toString(configBaseCmd, true)); duplicateCheck(configBaseCmd.getModelId()); ChatConfig chaConfig = chatConfigHelper.newChatConfig(configBaseCmd, user); - Long id = chatConfigRepository.createConfig(chaConfig); - applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); - return id; + return chatConfigRepository.createConfig(chaConfig); } private void duplicateCheck(Long modelId) { @@ -106,7 +96,6 @@ public class ConfigServiceImpl implements ConfigService { } ChatConfig chaConfig = chatConfigHelper.editChatConfig(configEditCmd, user); chatConfigRepository.updateConfig(chaConfig); - applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); return configEditCmd.getId(); } @@ -350,15 +339,7 @@ public class ConfigServiceImpl implements ConfigService { @Override public List getAllChatRichConfig() { - List chatConfigRichInfoList = new ArrayList<>(); - List modelSchemas = semanticInterpreter.getDataSetSchema(); - modelSchemas.stream().forEach(modelSchema -> { - ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getDataSet().getId()); - if (Objects.nonNull(chatConfigRichInfo)) { - chatConfigRichInfoList.add(chatConfigRichInfo); - } - }); - return chatConfigRichInfoList; + return new ArrayList<>(); } @Override @@ -367,4 +348,5 @@ public class ConfigServiceImpl implements ConfigService { return allChatRichConfig.stream() .collect(Collectors.toMap(ChatConfigRichResp::getModelId, value -> value, (k1, k2) -> k1)); } + } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java index 6353004a2..ee02f0016 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java @@ -4,11 +4,11 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; -import com.tencent.supersonic.chat.core.plugin.event.PluginAddEvent; -import com.tencent.supersonic.chat.core.plugin.event.PluginDelEvent; -import com.tencent.supersonic.chat.core.plugin.event.PluginUpdateEvent; +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginParseConfig; +import com.tencent.supersonic.chat.server.plugin.event.PluginAddEvent; +import com.tencent.supersonic.chat.server.plugin.event.PluginDelEvent; +import com.tencent.supersonic.chat.server.plugin.event.PluginUpdateEvent; import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository; import com.tencent.supersonic.chat.server.service.PluginService; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java deleted file mode 100644 index 15467045f..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.tencent.supersonic.chat.server.service.impl; - - -import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; -import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; -import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; -import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; -import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.chat.server.service.RecommendService; -import com.tencent.supersonic.chat.server.service.SemanticService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/*** - * Recommend Service impl - */ -@Service -@Slf4j -public class RecommendServiceImpl implements RecommendService { - - @Autowired - private ConfigService configService; - @Autowired - private SemanticService semanticService; - - @Override - public RecommendResp recommend(RecommendReq recommendReq, Long limit) { - if (Objects.isNull(limit) || limit <= 0) { - limit = Long.MAX_VALUE; - } - Long modelId = recommendReq.getModelId(); - if (Objects.isNull(modelId)) { - return new RecommendResp(); - } - DataSetSchema modelSchema = semanticService.getDataSetSchema(modelId); - if (Objects.isNull(modelSchema)) { - return new RecommendResp(); - } - List drillDownDimensions = Lists.newArrayList(); - Set metricElements = modelSchema.getMetrics(); - if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) { - Optional metric = metricElements.stream().filter(schemaElement -> - recommendReq.getMetricId().equals(schemaElement.getId()) - && !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements())) - .findFirst(); - if (metric.isPresent()) { - drillDownDimensions = metric.get().getRelatedSchemaElements().stream() - .map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()); - } - } - final List drillDownDimensionsFinal = drillDownDimensions; - List dimensions = modelSchema.getDimensions().stream() - .filter(dim -> { - if (Objects.isNull(dim)) { - return false; - } - if (!CollectionUtils.isEmpty(drillDownDimensionsFinal)) { - return drillDownDimensionsFinal.contains(dim.getId()); - } else { - return Objects.nonNull(dim.getUseCnt()); - } - }) - .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(limit) - .map(dimSchemaDesc -> { - SchemaElement item = new SchemaElement(); - item.setDataSet(modelId); - item.setName(dimSchemaDesc.getName()); - item.setBizName(dimSchemaDesc.getBizName()); - item.setId(dimSchemaDesc.getId()); - item.setAlias(dimSchemaDesc.getAlias()); - return item; - }).collect(Collectors.toList()); - - List metrics = modelSchema.getMetrics().stream() - .filter(metric -> Objects.nonNull(metric) && Objects.nonNull(metric.getUseCnt())) - .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(limit) - .map(metricSchemaDesc -> { - SchemaElement item = new SchemaElement(); - item.setDataSet(modelId); - item.setName(metricSchemaDesc.getName()); - item.setBizName(metricSchemaDesc.getBizName()); - item.setId(metricSchemaDesc.getId()); - item.setAlias(metricSchemaDesc.getAlias()); - return item; - }).collect(Collectors.toList()); - - RecommendResp response = new RecommendResp(); - response.setDimensions(dimensions); - response.setMetrics(metrics); - return response; - } - - @Override - public RecommendResp recommendMetricMode(RecommendReq recommendReq, Long limit) { - return recommend(recommendReq, limit); - } - - @Override - public List recommendQuestion(Long modelId) { - List recommendQuestions = new ArrayList<>(); - ChatConfigFilter chatConfigFilter = new ChatConfigFilter(); - chatConfigFilter.setModelId(modelId); - List chatConfigRespList = configService.search(chatConfigFilter, null); - if (!CollectionUtils.isEmpty(chatConfigRespList)) { - chatConfigRespList.stream().forEach(chatConfigResp -> { - if (Objects.nonNull(chatConfigResp) - && !CollectionUtils.isEmpty(chatConfigResp.getRecommendedQuestions())) { - recommendQuestions.add( - new RecommendQuestionResp(chatConfigResp.getModelId(), - chatConfigResp.getRecommendedQuestions())); - } - }); - return recommendQuestions; - } - return new ArrayList<>(); - } - - private List filterBlackItem(List itemList, List blackDimIdList) { - if (CollectionUtils.isEmpty(blackDimIdList) || CollectionUtils.isEmpty(itemList)) { - return itemList; - } - - return itemList.stream().filter(dim -> !blackDimIdList.contains(dim.getId())).collect(Collectors.toList()); - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java deleted file mode 100644 index 0fca0abe6..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.tencent.supersonic.chat.server.service.impl; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.util.concurrent.TimeUnit; - -@Service -@Slf4j -public class SchemaService { - - - public static final String ALL_CACHE = "all"; - private static final Integer META_CACHE_TIME = 30; - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - - private LoadingCache cache = CacheBuilder.newBuilder() - .expireAfterWrite(META_CACHE_TIME, TimeUnit.SECONDS) - .build( - new CacheLoader() { - @Override - public SemanticSchema load(String key) { - log.info("load getDomainSchemaInfo cache [{}]", key); - return new SemanticSchema(semanticInterpreter.getDataSetSchema()); - } - } - ); - - public DataSetSchema getDataSetSchema(Long id) { - return semanticInterpreter.getDataSetSchema(id, true); - } - - public SemanticSchema getSemanticSchema() { - return cache.getUnchecked(ALL_CACHE); - } - - public LoadingCache getCache() { - return cache; - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java index 4b3388284..028dc1e20 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; -import com.tencent.supersonic.chat.server.persistence.repository.StatisticsRepository; +import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.server.service.StatisticsService; +import com.tencent.supersonic.headless.server.persistence.mapper.StatisticsMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; @@ -15,11 +15,11 @@ import java.util.List; public class StatisticsServiceImpl implements StatisticsService { @Autowired - private StatisticsRepository statisticsRepository; + private StatisticsMapper statisticsMapper; @Async @Override public void batchSaveStatistics(List list) { - statisticsRepository.batchSaveStatistics(list); + statisticsMapper.batchSaveStatistics(list); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java index a460c7b6f..5b8735d48 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.util; import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java index f3609398f..1242c6270 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java @@ -1,43 +1,21 @@ package com.tencent.supersonic.chat.server.util; -import com.tencent.supersonic.chat.core.corrector.SemanticCorrector; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.mapper.SchemaMapper; -import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor; -import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; +import com.tencent.supersonic.headless.server.processor.ResultProcessor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.core.io.support.SpringFactoriesLoader; +import java.util.ArrayList; +import java.util.List; + @Slf4j public class ComponentFactory { - - private static List schemaMappers = new ArrayList<>(); - private static List semanticParsers = new ArrayList<>(); - private static List semanticCorrectors = new ArrayList<>(); - private static SemanticInterpreter semanticInterpreter; - private static List parseProcessors = new ArrayList<>(); + private static List parseProcessors = new ArrayList<>(); private static List executeProcessors = new ArrayList<>(); - public static List getSchemaMappers() { - return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers; - } - - public static List getSemanticParsers() { - return CollectionUtils.isEmpty(semanticParsers) ? init(SemanticParser.class, semanticParsers) : semanticParsers; - } - - public static List getSemanticCorrectors() { - return CollectionUtils.isEmpty(semanticCorrectors) ? init(SemanticCorrector.class, - semanticCorrectors) : semanticCorrectors; - } - - public static List getParseProcessors() { - return CollectionUtils.isEmpty(parseProcessors) ? init(ParseResultProcessor.class, + public static List getParseProcessors() { + return CollectionUtils.isEmpty(parseProcessors) ? init(ResultProcessor.class, parseProcessors) : parseProcessors; } @@ -46,13 +24,6 @@ public class ComponentFactory { ? init(ExecuteResultProcessor.class, executeProcessors) : executeProcessors; } - public static SemanticInterpreter getSemanticLayer() { - if (Objects.isNull(semanticInterpreter)) { - semanticInterpreter = init(SemanticInterpreter.class); - } - return semanticInterpreter; - } - private static List init(Class factoryType, List list) { list.addAll(SpringFactoriesLoader.loadFactories(factoryType, Thread.currentThread().getContextClassLoader())); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityEvent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityEvent.java deleted file mode 100644 index 98e267d3d..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityEvent.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.supersonic.chat.server.util; - -import com.tencent.supersonic.chat.server.config.ChatConfig; -import org.springframework.context.ApplicationEvent; - -public class VisibilityEvent extends ApplicationEvent { - private static final long serialVersionUID = 1L; - private ChatConfig chatConfig; - - public VisibilityEvent(Object source, ChatConfig chatConfig) { - super(source); - this.chatConfig = chatConfig; - } - - public void setChatConfig(ChatConfig chatConfig) { - this.chatConfig = chatConfig; - } - - public ChatConfig getChatConfig() { - return chatConfig; - } -} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityListener.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityListener.java deleted file mode 100644 index 11f5a0dfd..000000000 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/VisibilityListener.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.supersonic.chat.server.util; - -import com.github.benmanes.caffeine.cache.Cache; -import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; -import com.tencent.supersonic.chat.server.service.ConfigService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class VisibilityListener implements ApplicationListener { - - @Autowired - @Qualifier("searchCaffeineCache") - private Cache caffeineCache; - - @Autowired - private ConfigService configService; - - @Override - public void onApplicationEvent(VisibilityEvent event) { - log.info("visibility has changed,so update cache!"); - ItemNameVisibilityInfo itemNameVisibility = configService.getItemNameVisibility(event.getChatConfig()); - log.info("itemNameVisibility :{}", itemNameVisibility); - caffeineCache.put(event.getChatConfig().getModelId(), itemNameVisibility); - } -} diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/ChatBizLauncher.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/ChatBizLauncher.java deleted file mode 100644 index 146080b3d..000000000 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/ChatBizLauncher.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.chat.server.test; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; - - -@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic.chat"}) -@ComponentScan("com.tencent.supersonic.chat") -@MapperScan("com.tencent.supersonic.chat") -public class ChatBizLauncher { - - public static void main(String[] args) { - SpringApplication.run(ChatBizLauncher.class, args); - } - -} diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/MockBeansConfiguration.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/MockBeansConfiguration.java deleted file mode 100644 index f2d3139a8..000000000 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/MockBeansConfiguration.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.tencent.supersonic.chat.server.test.context; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.when; - -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; -import com.tencent.supersonic.chat.core.config.DefaultMetric; -import com.tencent.supersonic.chat.core.config.DefaultMetricInfo; -import com.tencent.supersonic.chat.core.config.EntityInternalDetail; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper; -import com.tencent.supersonic.chat.server.persistence.repository.impl.ChatContextRepositoryImpl; -import com.tencent.supersonic.chat.server.service.ChatService; -import com.tencent.supersonic.chat.server.service.QueryService; -import com.tencent.supersonic.chat.server.service.impl.ConfigServiceImpl; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; -import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ModelService; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.mockito.Mockito; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; - -@Configuration -public class MockBeansConfiguration { - - public static void getOrCreateContextMock(ChatService chatService) { - ChatContext context = new ChatContext(); - context.setChatId(1); - when(chatService.getOrCreateContext(1)).thenReturn(context); - } - - public static void buildHttpSemanticServiceImpl(List dimensionDescs, - List metricDescs) { - DefaultMetric defaultMetricDesc = new DefaultMetric(); - defaultMetricDesc.setUnit(3); - defaultMetricDesc.setPeriod(Constants.DAY); - List dimensionDescs1 = new ArrayList<>(); - DimSchemaResp dimensionDesc = new DimSchemaResp(); - dimensionDesc.setId(162L); - dimensionDescs1.add(dimensionDesc); - - DimSchemaResp dimensionDesc2 = new DimSchemaResp(); - dimensionDesc2.setId(163L); - dimensionDesc2.setBizName("song_name"); - dimensionDesc2.setName("歌曲名"); - - EntityInternalDetail entityInternalDetailDesc = new EntityInternalDetail(); - entityInternalDetailDesc.setDimensionList(new ArrayList<>(Arrays.asList(dimensionDesc2))); - MetricSchemaResp metricDesc = new MetricSchemaResp(); - metricDesc.setId(877L); - metricDesc.setBizName("js_play_cnt"); - metricDesc.setName("结算播放量"); - entityInternalDetailDesc.setMetricList(new ArrayList<>(Arrays.asList(metricDesc))); - - ModelSchemaResp modelSchemaDesc = new ModelSchemaResp(); - modelSchemaDesc.setDimensions(dimensionDescs); - modelSchemaDesc.setMetrics(metricDescs); - - } - - public static void getModelExtendMock(ConfigServiceImpl configService) { - DefaultMetricInfo defaultMetricInfo = new DefaultMetricInfo(); - defaultMetricInfo.setUnit(3); - defaultMetricInfo.setPeriod(Constants.DAY); - List defaultMetricInfos = new ArrayList<>(); - defaultMetricInfos.add(defaultMetricInfo); - - ChatConfigResp chaConfigDesc = new ChatConfigResp(); - when(configService.fetchConfigByModelId(anyLong())).thenReturn(chaConfigDesc); - } - - public static void dimensionDescBuild(DimensionService dimensionService, List dimensionDescs) { - when(dimensionService.getDimensions(any(DimensionFilter.class))).thenReturn(dimensionDescs); - } - - public static void metricDescBuild(MetricService metricService, List metricDescs) { - when(metricService.getMetrics(any(MetaFilter.class))).thenReturn(metricDescs); - } - - public static DimSchemaResp getDimensionDesc(Long id, String bizName, String name) { - DimSchemaResp dimensionDesc = new DimSchemaResp(); - dimensionDesc.setId(id); - dimensionDesc.setName(name); - dimensionDesc.setBizName(bizName); - return dimensionDesc; - } - - public static MetricSchemaResp getMetricDesc(Long id, String bizName, String name) { - MetricSchemaResp dimensionDesc = new MetricSchemaResp(); - dimensionDesc.setId(id); - dimensionDesc.setName(name); - dimensionDesc.setBizName(bizName); - return dimensionDesc; - } - - @Bean - public ChatContextRepositoryImpl getChatContextRepository() { - return Mockito.mock(ChatContextRepositoryImpl.class); - } - - @Bean - public QueryService getQueryService() { - return Mockito.mock(QueryService.class); - } - - @Bean - public DimensionService getDimensionService() { - return Mockito.mock(DimensionService.class); - } - - @Bean - public MetricService getMetricService() { - return Mockito.mock(MetricService.class); - } - - //queryDimensionDescs - - @Bean - public ModelService getModelService() { - return Mockito.mock(ModelService.class); - } - - @Bean - public ChatContextMapper getChatContextMapper() { - return Mockito.mock(ChatContextMapper.class); - } - - @Bean - public ConfigServiceImpl getModelExtendService() { - return Mockito.mock(ConfigServiceImpl.class); - } - - @Bean - public RestTemplate restTemplate() { - return new RestTemplate(); - } -} diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java deleted file mode 100644 index 1ab50b8f3..000000000 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java +++ /dev/null @@ -1,118 +0,0 @@ -package com.tencent.supersonic.chat.server.test.context; - -import com.google.gson.Gson; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; -import com.tencent.supersonic.common.pojo.DateConf; - -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import lombok.Data; - -public class SemanticParseObjectHelper { - - public static SemanticParseInfo copy(SemanticParseInfo semanticParseInfo) { - Gson g = new Gson(); - return g.fromJson(g.toJson(semanticParseInfo), SemanticParseInfo.class); - } - - public static SemanticParseInfo getSemanticParseInfo(String json) { - Gson gson = new Gson(); - SemanticParseJson semanticParseJson = gson.fromJson(json, SemanticParseJson.class); - if (semanticParseJson != null) { - return getSemanticParseInfo(semanticParseJson); - } - return null; - } - - private static SemanticParseInfo getSemanticParseInfo(SemanticParseJson semanticParseJson) { - Long model = semanticParseJson.getModel(); - Set dimensionList = new LinkedHashSet(); - Set metricList = new LinkedHashSet(); - Set chatFilters = new LinkedHashSet(); - - if (semanticParseJson.getFilter() != null && semanticParseJson.getFilter().size() > 0) { - for (List filter : semanticParseJson.getFilter()) { - chatFilters.add(getChatFilter(filter)); - } - } - - for (String dim : semanticParseJson.getDimensions()) { - dimensionList.add(getDimension(dim, model)); - } - for (String metric : semanticParseJson.getMetrics()) { - metricList.add(getMetric(metric, model)); - } - - SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - - semanticParseInfo.setDimensionFilters(chatFilters); - semanticParseInfo.setAggType(semanticParseJson.getAggregateType()); - semanticParseInfo.setQueryMode(semanticParseJson.getQueryMode()); - semanticParseInfo.setMetrics(metricList); - semanticParseInfo.setDimensions(dimensionList); - - DateConf dateInfo = getDateInfoAgo(semanticParseJson.getDay()); - semanticParseInfo.setDateInfo(dateInfo); - return semanticParseInfo; - } - - private static DateConf getDateInfoAgo(int dayAgo) { - if (dayAgo > 0) { - DateConf dateInfo = new DateConf(); - dateInfo.setUnit(dayAgo); - dateInfo.setDateMode(DateConf.DateMode.RECENT); - return dateInfo; - } - return null; - } - - private static QueryFilter getChatFilter(List filters) { - if (filters.size() > 1) { - QueryFilter chatFilter = new QueryFilter(); - - chatFilter.setBizName(filters.get(1)); - chatFilter.setOperator(FilterOperatorEnum.getSqlOperator(filters.get(2))); - if (filters.size() > 4) { - List valuse = new ArrayList<>(); - valuse.addAll(filters.subList(3, filters.size())); - chatFilter.setValue(valuse); - } else { - chatFilter.setValue(filters.get(3)); - } - - return chatFilter; - } - return null; - } - - private static SchemaElement getMetric(String bizName, Long modelId) { - SchemaElement metric = new SchemaElement(); - metric.setBizName(bizName); - return metric; - } - - private static SchemaElement getDimension(String bizName, Long modelId) { - SchemaElement dimension = new SchemaElement(); - dimension.setBizName(bizName); - return dimension; - } - - @Data - public static class SemanticParseJson { - - private Long model; - private String queryMode; - private AggregateTypeEnum aggregateType; - private Integer day; - private List dimensions; - private List metrics; - private List> filter; - - } -} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataInfo.java similarity index 82% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataInfo.java index 04576e892..627d60de6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetInfo.java similarity index 79% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetInfo.java index 1433f051a..64775d13c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataSetInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java similarity index 87% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java index c256d0813..46a94abd5 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java @@ -1,15 +1,11 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.QueryConfig; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import lombok.Data; + import java.util.HashSet; import java.util.Optional; import java.util.Set; -import lombok.Data; @Data public class DataSetSchema { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java similarity index 84% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java index f261a9840..c9a3cbe26 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java similarity index 77% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java index b205c87cb..e23ab11aa 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java @@ -1,6 +1,5 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java similarity index 95% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java index 2a1b0f24e..483b2a428 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaMapInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; import com.google.common.collect.Lists; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticCorrectInfo.java similarity index 73% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticCorrectInfo.java index 1099273dd..5d03de42a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticCorrectInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticCorrectInfo.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java similarity index 89% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index 9d0f68f8f..62a7edf1b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -1,15 +1,11 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; - -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.FilterType; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import lombok.Data; import java.util.ArrayList; @@ -40,10 +36,9 @@ public class SemanticParseInfo { private double score; private List elementMatches = new ArrayList<>(); private Map properties = new HashMap<>(); - private EntityInfo entityInfo; private SqlInfo sqlInfo = new SqlInfo(); private QueryType queryType = QueryType.ID; - + private EntityInfo entityInfo; private static class SchemaNameLengthComparator implements Comparator { @Override diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java similarity index 97% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java index d9ee7949f..79d083504 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java @@ -1,7 +1,5 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.headless.api.pojo; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import org.springframework.util.CollectionUtils; import java.io.Serializable; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java similarity index 72% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java index 62bc62c32..6d7b332c0 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo; import lombok.Data; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/CostType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/CostType.java new file mode 100644 index 000000000..95b0952e6 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/CostType.java @@ -0,0 +1,24 @@ +package com.tencent.supersonic.headless.api.pojo.enums; + +public enum CostType { + MAPPER(1, "mapper"), + PARSER(2, "parser"), + QUERY(3, "query"), + PROCESSOR(4, "processor"); + + private Integer type; + private String name; + + CostType(Integer type, String name) { + this.type = type; + this.name = name; + } + + public Integer getType() { + return type; + } + + public String getName() { + return name; + } +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java similarity index 69% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryType.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java index 2d54b99ee..cc884cbb4 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/QueryMethod.java @@ -1,19 +1,19 @@ package com.tencent.supersonic.headless.api.pojo.enums; -public enum QueryType { +public enum QueryMethod { SQL("SQL"), STRUCT("STRUCT"); private String value; - QueryType(String value) { + QueryMethod(String value) { this.value = value; } - public static QueryType of(String src) { - for (QueryType operatorEnum : QueryType.values()) { + public static QueryMethod of(String src) { + for (QueryMethod operatorEnum : QueryMethod.values()) { if (src.toUpperCase().contains(operatorEnum.value)) { return operatorEnum; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java similarity index 65% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java index 9cc96826a..98c2237f2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExecuteQueryReq.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.chat.api.pojo.request; +package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Builder; import lombok.Data; @@ -10,11 +10,9 @@ import lombok.Data; @Data public class ExecuteQueryReq { private User user; - private Integer agentId; + private Long queryId; private Integer chatId; private String queryText; - private Long queryId; - private Integer parseId; private SemanticParseInfo parseInfo; private boolean saveAnswer; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExplainSqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExplainSqlReq.java index 5502e61ce..42a59f277 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExplainSqlReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ExplainSqlReq.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.request; -import com.tencent.supersonic.headless.api.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -14,7 +14,7 @@ import lombok.ToString; @NoArgsConstructor public class ExplainSqlReq { - private QueryType queryTypeEnum; + private QueryMethod queryTypeEnum; private T queryReq; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java similarity index 78% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java index 5091bc6a2..0b67f66a2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataReq.java @@ -1,12 +1,14 @@ -package com.tencent.supersonic.chat.api.pojo.request; +package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import lombok.Data; + import java.util.HashSet; import java.util.Set; -import lombok.Data; @Data public class QueryDataReq { @@ -17,5 +19,5 @@ public class QueryDataReq { private Set metricFilters = new HashSet<>(); private DateConf dateInfo; private Long queryId; - private Integer parseId; + private SemanticParseInfo parseInfo; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java similarity index 95% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilter.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java index 35e753ac3..818194b02 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.request; +package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.base.Objects; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilters.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java similarity index 82% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilters.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java index 903288cd1..b6d6aa028 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilters.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilters.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.request; +package com.tencent.supersonic.headless.api.pojo.request; import lombok.Data; import java.util.ArrayList; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java new file mode 100644 index 000000000..8f3f8e9c9 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java @@ -0,0 +1,17 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.google.common.collect.Sets; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import lombok.Data; + +import java.util.Set; + +@Data +public class QueryReq { + private String queryText; + private Integer chatId; + private Set dataSetIds = Sets.newHashSet(); + private User user; + private QueryFilters queryFilters; + private boolean saveAnswer = true; +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java similarity index 66% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java index 83a0e9367..1202752fc 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseResp.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; import java.util.List; @@ -12,7 +12,7 @@ public class ParseResp { private Long queryId; private ParseState state; private List selectedParses = Lists.newArrayList(); - private ParseTimeCostDO parseTimeCost = new ParseTimeCostDO(); + private ParseTimeCostResp parseTimeCost = new ParseTimeCostResp(); public enum ParseState { COMPLETED, diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseTimeCostResp.java similarity index 59% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseTimeCostResp.java index 7049c8e5c..4b1df4e45 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ParseTimeCostResp.java @@ -1,15 +1,15 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; import lombok.Data; @Data -public class ParseTimeCostDO { +public class ParseTimeCostResp { private long parseStartTime; private long parseTime; private long sqlTime; - public ParseTimeCostDO() { + public ParseTimeCostResp() { this.parseStartTime = System.currentTimeMillis(); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java similarity index 67% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java index 2d88aaffd..473b35d1a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResp.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; import java.util.Date; import java.util.List; + @Data public class QueryResp { @@ -16,7 +17,5 @@ public class QueryResp { private String queryText; private QueryResult queryResult; private List parseInfos; - private List similarQueries; - } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java similarity index 66% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java index 4cce89735..b85570c27 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryResult.java @@ -1,18 +1,16 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; + import java.util.List; import java.util.Map; @Data public class QueryResult { - - public EntityInfo entityInfo; - public AggregateInfo aggregateInfo; private Long queryId; private String queryMode; private String querySql; @@ -23,5 +21,5 @@ public class QueryResult { private Object response; private List> queryResults; private Long queryTimeCost; - private List recommendedDimensions; + private EntityInfo entityInfo; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryState.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryState.java similarity index 59% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryState.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryState.java index 30ecc1e2c..9b4c39703 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryState.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/QueryState.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; public enum QueryState { SUCCESS, diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java similarity index 80% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java index 5a1fae483..da67ed3ac 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResp.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; import java.util.List; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java similarity index 94% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java index 4e7b251de..742abaa4f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SearchResult.java @@ -1,12 +1,13 @@ -package com.tencent.supersonic.chat.api.pojo.response; +package com.tencent.supersonic.headless.api.pojo.response; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import java.util.Objects; import lombok.Builder; import lombok.Data; import lombok.Getter; import lombok.Setter; +import java.util.Objects; + @Data @Setter @Getter diff --git a/headless/core/pom.xml b/headless/core/pom.xml index 26a5e8e25..327998c32 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -29,7 +29,28 @@ ST4 ${st.version} - + + org.testng + testng + ${org.testng.version} + test + + + junit + junit + test + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-inline + ${mockito-inline.version} + test + org.springframework spring-context @@ -140,6 +161,12 @@ ${commons.compress.version} compile + + + com.github.xkzhangsan + xk-time + ${xk.time.version} + diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/BaseSemanticCorrector.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/BaseSemanticCorrector.java index 0c9078a4c..c50905f03 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/BaseSemanticCorrector.java @@ -1,15 +1,15 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GrammarCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GrammarCorrector.java similarity index 81% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GrammarCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GrammarCorrector.java index 8fdaf3ac5..fc22f12ea 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GrammarCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GrammarCorrector.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; + +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import lombok.extern.slf4j.Slf4j; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import java.util.ArrayList; import java.util.List; -import lombok.extern.slf4j.Slf4j; /** * Correcting SQL syntax, primarily including fixes to select, where, groupBy, and Having clauses diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GroupByCorrector.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GroupByCorrector.java index 12d891b6c..555a91856 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GroupByCorrector.java @@ -1,26 +1,17 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; -import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.DataSetService; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * Perform SQL corrections on the "Group by" section in S2SQL. @@ -39,23 +30,6 @@ public class GroupByCorrector extends BaseSemanticCorrector { private Boolean needAddGroupBy(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { Long dataSetId = semanticParseInfo.getDataSetId(); - DataSetService dataSetService = ContextUtils.getBean(DataSetService.class); - ModelService modelService = ContextUtils.getBean(ModelService.class); - DataSetResp dataSetResp = dataSetService.getDataSet(dataSetId); - List modelIds = dataSetResp.getDataSetDetail() - .getDataSetModelConfigs().stream().map(DataSetModelConfig::getId) - .collect(Collectors.toList()); - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setIds(modelIds); - List modelRespList = modelService.getModelList(metaFilter); - for (ModelResp modelResp : modelRespList) { - List dimList = modelResp.getModelDetail().getDimensions(); - for (Dim dim : dimList) { - if (Objects.nonNull(dim.getTypeParams()) && dim.getTypeParams().getTimeGranularity().equals("none")) { - return false; - } - } - } //add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectS2SQL(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/HavingCorrector.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/HavingCorrector.java index 02b2d381b..2a60ff535 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/HavingCorrector.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import org.apache.commons.lang3.StringUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SchemaCorrector.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SchemaCorrector.java index b21829fa1..3275622bc 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SchemaCorrector.java @@ -1,11 +1,5 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; @@ -16,6 +10,12 @@ import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.core.chat.parser.llm.ParseResult; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.ElementValue; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SelectCorrector.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SelectCorrector.java index 6e088d508..796e7bd2a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SelectCorrector.java @@ -1,12 +1,14 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; + -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; -import java.util.List; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; +import java.util.List; + /** * Perform SQL corrections on the "Select" section in S2SQL. */ diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SemanticCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SemanticCorrector.java similarity index 57% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SemanticCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SemanticCorrector.java index 1a53484e0..49321fc92 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SemanticCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/SemanticCorrector.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; + +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; /** * A semantic corrector checks validity of extracted semantic information and diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java index f39860601..83ad22ff6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java @@ -1,17 +1,19 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; + -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.util.jsqlparser.DateVisitor.DateBoundInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlDateSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; -import java.util.Objects; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + /** * Perform SQL corrections on the time in S2SQL. */ diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java similarity index 94% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java index a0bcc5590..48cb1110e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java @@ -1,18 +1,19 @@ -package com.tencent.supersonic.chat.core.corrector; +package com.tencent.supersonic.headless.core.chat.corrector; + -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.utils.S2SqlDateHelper; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.utils.S2SqlDateHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMapper.java similarity index 90% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMapper.java index 2d6b001e4..b5af9d979 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMapper.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMatchStrategy.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMatchStrategy.java index 5970d98bf..84d6f3bcd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMatchStrategy.java @@ -1,8 +1,14 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; + -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -13,11 +19,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -104,7 +105,7 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } public List getMatches(QueryContext queryContext, List terms) { - Set dataSetIds = mapperHelper.getDataSetIds(queryContext.getDataSetId(), queryContext.getAgent()); + Set dataSetIds = queryContext.getDataSetIds(); terms = filterByDataSetId(terms, dataSetIds); Map> matchResult = match(queryContext, terms, dataSetIds); List matches = new ArrayList<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/DatabaseMatchStrategy.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/DatabaseMatchStrategy.java index 2712bc371..690938246 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/DatabaseMatchStrategy.java @@ -1,12 +1,13 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; + -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -37,7 +38,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy @Override public Map> match(QueryContext queryContext, List terms, - Set detectDataSetIds) { + Set detectDataSetIds) { this.allElements = getSchemaElements(queryContext); return super.match(queryContext, terms, detectDataSetIds); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMapper.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMapper.java index 2892defac..98690784a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMapper.java @@ -1,17 +1,18 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; + -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; -import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; + import java.util.List; import java.util.Objects; @@ -25,8 +26,7 @@ public class EmbeddingMapper extends BaseMapper { public void doMap(QueryContext queryContext) { //1. query from embedding by queryText String queryText = queryContext.getQueryText(); - KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class); - List terms = knowledgeService.getTerms(queryText); + List terms = HanlpHelper.getTerms(queryText, queryContext.getModelIdToDataSetIds()); EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class); List matchResults = matchStrategy.getMatches(queryContext, terms); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMatchStrategy.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMatchStrategy.java index b359fecc9..ad23cd3d0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMatchStrategy.java @@ -1,20 +1,14 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; -import com.tencent.supersonic.headless.server.service.MetaEmbeddingService; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; +import com.tencent.supersonic.headless.core.knowledge.MetaEmbeddingService; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -22,6 +16,12 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + /** * EmbeddingMatchStrategy uses vector database to perform * similarity search against the embeddings of schema elements. @@ -49,7 +49,7 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { @Override public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, - String detectSegment, int offset) { + String detectSegment, int offset) { } @@ -68,19 +68,19 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy { optimizationConfig.getEmbeddingMapperBatch()); for (List queryTextsSub : queryTextsSubList) { - detectByQueryTextsSub(results, detectDataSetIds, queryTextsSub); + detectByQueryTextsSub(results, detectDataSetIds, queryTextsSub, queryContext.getModelIdToDataSetIds()); } } private void detectByQueryTextsSub(Set results, Set detectDataSetIds, - List queryTextsSub) { + List queryTextsSub, Map> modelIdToDataSetIds) { int embeddingNumber = optimizationConfig.getEmbeddingMapperNumber(); Double distance = optimizationConfig.getEmbeddingMapperDistanceThreshold(); // step1. build query params RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(queryTextsSub).build(); // step2. retrieveQuery by detectSegment List retrieveQueryResults = metaEmbeddingService.retrieveQuery( - new ArrayList<>(detectDataSetIds), retrieveQuery, embeddingNumber); + retrieveQuery, embeddingNumber, modelIdToDataSetIds); if (CollectionUtils.isEmpty(retrieveQueryResults)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EntityMapper.java similarity index 89% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EntityMapper.java index 645d17d6d..a9bf8b131 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EntityMapper.java @@ -1,13 +1,12 @@ -package com.tencent.supersonic.chat.core.mapper; - +package com.tencent.supersonic.headless.core.chat.mapper; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java index af9d62f56..d02a47ac3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java @@ -1,11 +1,16 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -13,11 +18,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; /** * HanlpDictMatchStrategy uses HanLP to @@ -39,7 +39,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { @Override public Map> match(QueryContext queryContext, List terms, - Set detectDataSetIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; @@ -65,10 +65,12 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { // step1. pre search Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize(); LinkedHashSet hanlpMapResults = knowledgeService.prefixSearch(detectSegment, - oneDetectionMaxSize, detectDataSetIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); + oneDetectionMaxSize, queryContext.getModelIdToDataSetIds()) + .stream().collect(Collectors.toCollection(LinkedHashSet::new)); // step2. suffix search LinkedHashSet suffixHanlpMapResults = knowledgeService.suffixSearch(detectSegment, - oneDetectionMaxSize, detectDataSetIds).stream().collect(Collectors.toCollection(LinkedHashSet::new)); + oneDetectionMaxSize, queryContext.getModelIdToDataSetIds()) + .stream().collect(Collectors.toCollection(LinkedHashSet::new)); hanlpMapResults.addAll(suffixHanlpMapResults); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/KeywordMapper.java similarity index 92% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/KeywordMapper.java index b182395ec..ffd01ed96 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/KeywordMapper.java @@ -1,17 +1,16 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult; import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.headless.server.service.KnowledgeService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; -import com.tencent.supersonic.common.util.ContextUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -33,8 +32,7 @@ public class KeywordMapper extends BaseMapper { public void doMap(QueryContext queryContext) { String queryText = queryContext.getQueryText(); //1.hanlpDict Match - KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class); - List terms = knowledgeService.getTerms(queryText); + List terms = HanlpHelper.getTerms(queryText, queryContext.getModelIdToDataSetIds()); HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class); List hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MapperHelper.java similarity index 66% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MapperHelper.java index ceadebe81..00d49744e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MapperHelper.java @@ -1,9 +1,8 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; import com.hankcs.hanlp.algorithm.EditDistance; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -11,11 +10,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; @Data @@ -81,34 +78,4 @@ public class MapperHelper { return 1 - (double) EditDistance.compute(detectSegmentLower, matchNameLower) / Math.max(matchName.length(), detectSegment.length()); } - - public Set getDataSetIds(Long dataSetId, Agent agent) { - - Set detectDataSetIds = new HashSet<>(); - if (Objects.nonNull(agent)) { - detectDataSetIds = agent.getDataSetIds(); - } - //contains all - if (Agent.containsAllModel(detectDataSetIds)) { - if (Objects.nonNull(dataSetId) && dataSetId > 0) { - Set result = new HashSet<>(); - result.add(dataSetId); - return result; - } - return new HashSet<>(); - } - - if (Objects.nonNull(detectDataSetIds)) { - detectDataSetIds = detectDataSetIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); - } - - if (Objects.nonNull(dataSetId) && dataSetId > 0 && Objects.nonNull(detectDataSetIds)) { - if (detectDataSetIds.contains(dataSetId)) { - Set result = new HashSet<>(); - result.add(dataSetId); - return result; - } - } - return detectDataSetIds; - } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchStrategy.java similarity index 76% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchStrategy.java index b85f960c0..a283dde58 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchStrategy.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; + -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import java.util.List; import java.util.Map; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchText.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchText.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchText.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchText.java index 93487b682..bd8453e93 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchText.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MatchText.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; -import java.util.Objects; import lombok.Builder; import lombok.Data; import lombok.ToString; +import java.util.Objects; + @Data @ToString @Builder diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/ModelWithSemanticType.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/ModelWithSemanticType.java index 92c734557..7d0e8c696 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/ModelWithSemanticType.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import java.io.Serializable; import lombok.Data; import lombok.ToString; +import java.io.Serializable; + @Data @ToString public class ModelWithSemanticType implements Serializable { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/QueryFilterMapper.java similarity index 77% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/QueryFilterMapper.java index 37d19776b..9dcef4b6d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/QueryFilterMapper.java @@ -1,15 +1,14 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -26,23 +25,19 @@ public class QueryFilterMapper implements SchemaMapper { @Override public void map(QueryContext queryContext) { - Agent agent = queryContext.getAgent(); - if (agent == null || CollectionUtils.isEmpty(agent.getDataSetIds())) { + Set dataSetIds = queryContext.getDataSetIds(); + if (CollectionUtils.isEmpty(dataSetIds)) { return; } - if (Agent.containsAllModel(agent.getDataSetIds())) { - return; - } - Set viewIds = agent.getDataSetIds(); SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - clearOtherSchemaElementMatch(viewIds, schemaMapInfo); - for (Long viewId : viewIds) { - List schemaElementMatches = schemaMapInfo.getMatchedElements(viewId); + clearOtherSchemaElementMatch(dataSetIds, schemaMapInfo); + for (Long dataSetId : dataSetIds) { + List schemaElementMatches = schemaMapInfo.getMatchedElements(dataSetId); if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); - schemaMapInfo.setMatchedElements(viewId, schemaElementMatches); + schemaMapInfo.setMatchedElements(dataSetId, schemaElementMatches); } - addValueSchemaElementMatch(viewId, queryContext, schemaElementMatches); + addValueSchemaElementMatch(dataSetId, queryContext, schemaElementMatches); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SchemaMapper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SchemaMapper.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SchemaMapper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SchemaMapper.java index e31300496..c903ff45e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SchemaMapper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SchemaMapper.java @@ -1,6 +1,7 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; -import com.tencent.supersonic.chat.core.pojo.QueryContext; + +import com.tencent.supersonic.headless.core.pojo.QueryContext; /** * A schema mapper identifies references to schema elements(metrics/dimensions/entities/values) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SearchMatchStrategy.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SearchMatchStrategy.java index 0535f2bea..10a0ead48 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SearchMatchStrategy.java @@ -1,22 +1,23 @@ -package com.tencent.supersonic.chat.core.mapper; +package com.tencent.supersonic.headless.core.chat.mapper; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.core.knowledge.SearchService; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.SearchService; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; /** * SearchMatchStrategy encapsulates a concrete matching algorithm @@ -32,7 +33,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy { @Override public Map> match(QueryContext queryContext, List originals, - Set detectDataSetIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); Map regOffsetToLength = getRegOffsetToLength(originals); @@ -57,9 +58,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy { if (StringUtils.isNotEmpty(detectSegment)) { List hanlpMapResults = knowledgeService.prefixSearch(detectSegment, - SearchService.SEARCH_SIZE, detectDataSetIds); + SearchService.SEARCH_SIZE, queryContext.getModelIdToDataSetIds()); List suffixHanlpMapResults = knowledgeService.suffixSearch( - detectSegment, SEARCH_SIZE, detectDataSetIds); + detectSegment, SEARCH_SIZE, queryContext.getModelIdToDataSetIds()); hanlpMapResults.addAll(suffixHanlpMapResults); // remove entity name where search hanlpMapResults = hanlpMapResults.stream().filter(entry -> { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/JavaLLMProxy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/JavaLLMProxy.java new file mode 100644 index 000000000..ade4a0bf5 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/JavaLLMProxy.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.headless.core.chat.parser; + + +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.chat.parser.llm.SqlGeneration; +import com.tencent.supersonic.headless.core.chat.parser.llm.SqlGenerationFactory; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import dev.langchain4j.model.chat.ChatLanguageModel; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.Objects; + +/** + * LLMProxy based on langchain4j Java version. + */ +@Slf4j +@Component +public class JavaLLMProxy implements LLMProxy { + + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); + + @Override + public boolean isSkip(QueryContext queryContext) { + ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); + if (Objects.isNull(chatLanguageModel)) { + log.warn("chatLanguageModel is null, skip :{}", JavaLLMProxy.class.getName()); + return true; + } + return false; + } + + public LLMResp query2sql(LLMReq llmReq, Long dataSetId) { + + SqlGeneration sqlGeneration = SqlGenerationFactory.get( + SqlGenerationMode.getMode(llmReq.getSqlGenerationMode())); + String modelName = llmReq.getSchema().getDataSetName(); + LLMResp result = sqlGeneration.generation(llmReq, dataSetId); + result.setQuery(llmReq.getQueryText()); + result.setModelName(modelName); + return result; + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/LLMProxy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/LLMProxy.java new file mode 100644 index 000000000..f2910bd21 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/LLMProxy.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.headless.core.chat.parser; + + +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; + +/** + * LLMProxy encapsulates functions performed by LLMs so that multiple + * orchestration frameworks (e.g. LangChain in python, LangChain4j in java) + * could be used. + */ +public interface LLMProxy { + + boolean isSkip(QueryContext queryContext); + + LLMResp query2sql(LLMReq llmReq, Long dataSetId); + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/PythonLLMProxy.java similarity index 56% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/PythonLLMProxy.java index 2ed2b1bbe..8eb34bbb0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/PythonLLMProxy.java @@ -1,16 +1,12 @@ -package com.tencent.supersonic.chat.core.parser; +package com.tencent.supersonic.headless.core.chat.parser; -import com.alibaba.fastjson.JSON; -import com.tencent.supersonic.chat.core.config.LLMParserConfig; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionCallConfig; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; -import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.core.config.LLMParserConfig; +import com.tencent.supersonic.headless.core.chat.parser.llm.OutputFormat; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -23,9 +19,7 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; -import java.net.URI; import java.net.URL; import java.util.ArrayList; @@ -78,27 +72,4 @@ public class PythonLLMProxy implements LLMProxy { return null; } - public FunctionResp requestFunction(FunctionReq functionReq) { - FunctionCallConfig functionCallInfoConfig = ContextUtils.getBean(FunctionCallConfig.class); - String url = functionCallInfoConfig.getUrl() + functionCallInfoConfig.getPluginSelectPath(); - HttpHeaders headers = new HttpHeaders(); - long startTime = System.currentTimeMillis(); - headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity entity = new HttpEntity<>(JSON.toJSONString(functionReq), headers); - URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); - RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - try { - log.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq)); - keyPipelineLog.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq)); - ResponseEntity responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, - FunctionResp.class); - log.info("requestFunction responseEntity:{},cost:{}", responseEntity, - System.currentTimeMillis() - startTime); - keyPipelineLog.info("response:{}", responseEntity.getBody()); - return responseEntity.getBody(); - } catch (Exception e) { - log.error("requestFunction error", e); - } - return null; - } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/QueryTypeParser.java similarity index 72% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/QueryTypeParser.java index ba76579f9..eda8610b6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/QueryTypeParser.java @@ -1,13 +1,14 @@ -package com.tencent.supersonic.chat.core.parser; +package com.tencent.supersonic.headless.core.chat.parser; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import java.util.List; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; +import java.util.List; + /** * QueryTypeParser resolves query type as either METRIC or TAG, or ID. */ diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SatisfactionChecker.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SatisfactionChecker.java similarity index 79% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SatisfactionChecker.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SatisfactionChecker.java index a67d74dc4..34b491762 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SatisfactionChecker.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SatisfactionChecker.java @@ -1,12 +1,11 @@ -package com.tencent.supersonic.chat.core.parser; +package com.tencent.supersonic.headless.core.chat.parser; - -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery; import lombok.extern.slf4j.Slf4j; /** diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SemanticParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SemanticParser.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SemanticParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SemanticParser.java index bdb309bc1..8e57cb973 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/SemanticParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/SemanticParser.java @@ -1,8 +1,7 @@ -package com.tencent.supersonic.chat.core.parser; +package com.tencent.supersonic.headless.core.chat.parser; - -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; /** * A semantic parser understands user queries and extracts semantic information. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetMatchResult.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetMatchResult.java index ccb65e7af..54f59ec0d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetMatchResult.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetMatchResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetResolver.java similarity index 52% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetResolver.java index edde73b05..636ceed80 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/DataSetResolver.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/DataSetResolver.java @@ -1,7 +1,6 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import java.util.Set; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/HeuristicDataSetResolver.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/HeuristicDataSetResolver.java index efb8370f3..33fcdaf92 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicDataSetResolver.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/HeuristicDataSetResolver.java @@ -1,13 +1,12 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; - import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -39,7 +38,7 @@ public class HeuristicDataSetResolver implements DataSetResolver { return dataSetSelect; } } else { - Map.Entry maxDataSet = dataSetTypeMap.entrySet().stream() + Entry maxDataSet = dataSetTypeMap.entrySet().stream() .filter(entry -> dataSetQueryModes.containsKey(entry.getKey())) .sorted((o1, o2) -> { int difference = o2.getValue().getCount() - o1.getValue().getCount(); @@ -87,7 +86,7 @@ public class HeuristicDataSetResolver implements DataSetResolver { public static Map getDataSetTypeMap(SchemaMapInfo schemaMap) { Map dataSetCount = new HashMap<>(); - for (Map.Entry> entry : schemaMap.getDataSetElementMatches().entrySet()) { + for (Entry> entry : schemaMap.getDataSetElementMatches().entrySet()) { List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); if (schemaElementMatches != null && schemaElementMatches.size() > 0) { if (!dataSetCount.containsKey(entry.getKey())) { @@ -115,13 +114,6 @@ public class HeuristicDataSetResolver implements DataSetResolver { public Long resolve(QueryContext queryContext, Set agentDataSetIds) { SchemaMapInfo mapInfo = queryContext.getMapInfo(); Set matchedDataSets = mapInfo.getMatchedDataSetInfos(); - Long dataSetId = queryContext.getDataSetId(); - if (Objects.nonNull(dataSetId) && dataSetId > 0) { - if (CollectionUtils.isEmpty(agentDataSetIds) || agentDataSetIds.contains(dataSetId)) { - return dataSetId; - } - return null; - } if (CollectionUtils.isNotEmpty(agentDataSetIds)) { matchedDataSets.retainAll(agentDataSetIds); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/InputFormat.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/InputFormat.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/InputFormat.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/InputFormat.java index 43d6a2eef..cd36db063 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/InputFormat.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/InputFormat.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; + +import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; @Slf4j public class InputFormat { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMRequestService.java similarity index 79% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMRequestService.java index c5653f6c4..8d71f4cd2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMRequestService.java @@ -1,29 +1,25 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.utils.S2SqlDateHelper; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentToolType; -import com.tencent.supersonic.chat.core.agent.NL2SQLTool; -import com.tencent.supersonic.chat.core.config.LLMParserConfig; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.parser.SatisfactionChecker; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; +import com.tencent.supersonic.headless.core.chat.parser.SatisfactionChecker; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.ElementValue; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.config.LLMParserConfig; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.utils.ComponentFactory; +import com.tencent.supersonic.headless.core.utils.S2SqlDateHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -37,21 +33,22 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @Slf4j @Service public class LLMRequestService { - - protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); @Autowired private LLMParserConfig llmParserConfig; @Autowired private OptimizationConfig optimizationConfig; public boolean isSkip(QueryContext queryCtx) { + if (!queryCtx.isEnableLLM()) { + log.info("not enable llm, skip"); + return true; + } if (ComponentFactory.getLLMProxy().isSkip(queryCtx)) { return true; } @@ -63,38 +60,13 @@ public class LLMRequestService { } public Long getDataSetId(QueryContext queryCtx) { - Agent agent = queryCtx.getAgent(); - Set agentDataSetIds = new HashSet<>(); - if (Objects.nonNull(agent)) { - agentDataSetIds = agent.getDataSetIds(AgentToolType.NL2SQL_LLM); - } - if (Agent.containsAllModel(agentDataSetIds)) { - agentDataSetIds = new HashSet<>(); - } DataSetResolver dataSetResolver = ComponentFactory.getModelResolver(); - return dataSetResolver.resolve(queryCtx, agentDataSetIds); + return dataSetResolver.resolve(queryCtx, queryCtx.getDataSetIds()); } - public NL2SQLTool getParserTool(QueryContext queryCtx, Long dataSetId) { - Agent agent = queryCtx.getAgent(); - if (Objects.isNull(agent)) { - return null; - } - List commonAgentTools = agent.getParserTools(AgentToolType.NL2SQL_LLM); - Optional llmParserTool = commonAgentTools.stream() - .filter(tool -> { - List dataSetIds = tool.getDataSetIds(); - if (Agent.containsAllModel(new HashSet<>(dataSetIds))) { - return true; - } - return dataSetIds.contains(dataSetId); - }) - .findFirst(); - return llmParserTool.orElse(null); - } - - public LLMReq getLlmReq(QueryContext queryCtx, Long dataSetId, List linkingValues) { - Map dataSetIdToName = queryCtx.getSemanticSchema().getDataSetIdToName(); + public LLMReq getLlmReq(QueryContext queryCtx, Long dataSetId, + SemanticSchema semanticSchema, List linkingValues) { + Map dataSetIdToName = semanticSchema.getDataSetIdToName(); String queryText = queryCtx.getQueryText(); LLMReq llmReq = new LLMReq(); @@ -147,8 +119,8 @@ public class LLMRequestService { private String getPriorExts(Long dataSetId, List fieldNameList) { StringBuilder extraInfoSb = new StringBuilder(); - List dataSetSchemaResps = semanticInterpreter.fetchDataSetSchema( - Lists.newArrayList(dataSetId), true); + //todo + List dataSetSchemaResps = Lists.newArrayList(); if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { DataSetSchemaResp dataSetSchemaResp = dataSetSchemaResps.get(0); Map fieldNameToDataFormatType = dataSetSchemaResp.getMetrics() @@ -164,8 +136,7 @@ public class LLMRequestService { } } return result.stream(); - }) - .collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1)); + }).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); for (String fieldName : fieldNameList) { String dataFormatType = fieldNameToDataFormatType.get(fieldName); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMResponseService.java similarity index 74% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMResponseService.java index a9b3b0f84..7317f133c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMResponseService.java @@ -1,15 +1,14 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; -import com.tencent.supersonic.chat.core.agent.NL2SQLTool; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.jsqlparser.SqlEqualHelper; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.llm.LLMSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.springframework.stereotype.Service; @@ -29,14 +28,11 @@ public class LLMResponseService { LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); parseInfo.setDataSet(queryCtx.getSemanticSchema().getDataSet(parseResult.getDataSetId())); - NL2SQLTool commonAgentTool = parseResult.getCommonAgentTool(); parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getDataSetId())); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, parseResult); properties.put("type", "internal"); - properties.put("name", commonAgentTool.getName()); - parseInfo.setProperties(properties); parseInfo.setScore(queryCtx.getQueryText().length() * (1 + weight)); parseInfo.setQueryMode(semanticQuery.getQueryMode()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMSqlParser.java similarity index 62% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMSqlParser.java index 211c810d1..b95b73092 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMSqlParser.java @@ -1,19 +1,19 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; + -import com.tencent.supersonic.chat.core.agent.NL2SQLTool; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.MapUtils; @Slf4j public class LLMSqlParser implements SemanticParser { @@ -31,26 +31,20 @@ public class LLMSqlParser implements SemanticParser { if (dataSetId == null) { return; } - //3.get agent tool and determine whether to skip this parser. - NL2SQLTool commonAgentTool = requestService.getParserTool(queryCtx, dataSetId); - if (Objects.isNull(commonAgentTool)) { - log.info("no tool in this agent, skip {}", LLMSqlParser.class); - return; - } - //4.construct a request, call the API for the large model, and retrieve the results. - List linkingValues = requestService.getValueList(queryCtx, dataSetId); - LLMReq llmReq = requestService.getLlmReq(queryCtx, dataSetId, linkingValues); + //3.construct a request, call the API for the large model, and retrieve the results. + List linkingValues = requestService.getValueList(queryCtx, dataSetId); + SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); + LLMReq llmReq = requestService.getLlmReq(queryCtx, dataSetId, semanticSchema, linkingValues); LLMResp llmResp = requestService.requestLLM(llmReq, dataSetId); if (Objects.isNull(llmResp)) { return; } - //5. deduplicate the SQL result list and build parserInfo + //4. deduplicate the SQL result list and build parserInfo LLMResponseService responseService = ContextUtils.getBean(LLMResponseService.class); Map deduplicationSqlResp = responseService.getDeduplicationSqlResp(llmResp); ParseResult parseResult = ParseResult.builder() .dataSetId(dataSetId) - .commonAgentTool(commonAgentTool) .llmReq(llmReq) .llmResp(llmResp) .linkingValues(linkingValues) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSCSqlGeneration.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSCSqlGeneration.java index 11324108f..43325b79e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSCSqlGeneration.java @@ -1,11 +1,10 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSqlGeneration.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSqlGeneration.java index df35c6dfe..738583554 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OnePassSqlGeneration.java @@ -1,12 +1,11 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlResp; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OutputFormat.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OutputFormat.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OutputFormat.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OutputFormat.java index 66bf86ca6..cbbe093c1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OutputFormat.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/OutputFormat.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; + +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -12,8 +12,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; /*** * output format @@ -113,26 +111,13 @@ public class OutputFormat { return results; } - public static FunctionResp functionCallParse(String llmOutput) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(llmOutput); - String selectedTool = jsonNode.get("选择工具").asText(); - FunctionResp resp = new FunctionResp(); - resp.setToolSelection(selectedTool); - return resp; - } catch (Exception e) { - log.error("", e); - } - return null; - } - public static Map buildSqlRespMap(List> sqlExamples, - Map sqlMap) { + Map sqlMap) { return sqlMap.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, entry -> LLMSqlResp.builder().sqlWeight(entry.getValue()).fewShots(sqlExamples).build()) ); } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/ParseResult.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/ParseResult.java new file mode 100644 index 000000000..7a006e4b4 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/ParseResult.java @@ -0,0 +1,29 @@ +package com.tencent.supersonic.headless.core.chat.parser.llm; + +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.ElementValue; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ParseResult { + + private Long dataSetId; + + private LLMReq llmReq; + + private LLMResp llmResp; + + private QueryReq request; + + private List linkingValues; +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExamplarLoader.java similarity index 98% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExamplarLoader.java index 7a586e9d3..aa0a1d149 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExamplarLoader.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExamplarLoader.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; import com.fasterxml.jackson.core.type.TypeReference; @@ -10,6 +10,12 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; + import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -18,11 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; -import org.springframework.stereotype.Component; @Slf4j @Component diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExample.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExample.java similarity index 81% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExample.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExample.java index 9837c0e90..20eec3372 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlExample.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlExample.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGeneration.java similarity index 60% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGeneration.java index 48d266b49..d3f502548 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGeneration.java @@ -1,8 +1,7 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; /** * Sql Generation interface, generating SQL using a large model. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGenerationFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGenerationFactory.java similarity index 76% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGenerationFactory.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGenerationFactory.java index 72ea6795f..6fd07789b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGenerationFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlGenerationFactory.java @@ -1,7 +1,9 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; + +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; + import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlPromptGenerator.java similarity index 94% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlPromptGenerator.java index daf031825..0e95c15dc 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/SqlPromptGenerator.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.ElementValue; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Component; @@ -70,7 +70,7 @@ public class SqlPromptGenerator { } public List generatePromptPool(LLMReq llmReq, List>> exampleListPool, - boolean isSqlPrompt) { + boolean isSqlPrompt) { List promptPool = new ArrayList<>(); for (List> exampleList : exampleListPool) { String prompt; @@ -98,7 +98,7 @@ public class SqlPromptGenerator { public Pair transformQuestionPrompt(LLMReq llmReq) { String modelName = llmReq.getSchema().getDataSetName(); List fieldNameList = llmReq.getSchema().getFieldNameList(); - List linking = llmReq.getLinking(); + List linking = llmReq.getLinking(); String currentDate = llmReq.getCurrentDate(); String priorExts = llmReq.getPriorExts(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSCSqlGeneration.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSCSqlGeneration.java index 5803f2811..199ff5da7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSCSqlGeneration.java @@ -1,11 +1,10 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSqlGeneration.java similarity index 89% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSqlGeneration.java index 2866f2772..00da5dc11 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/TwoPassSqlGeneration.java @@ -1,11 +1,10 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; +package com.tencent.supersonic.headless.core.chat.parser.llm; - -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AggregateTypeParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/AggregateTypeParser.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AggregateTypeParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/AggregateTypeParser.java index f67f391fc..406588465 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AggregateTypeParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/AggregateTypeParser.java @@ -1,19 +1,13 @@ -package com.tencent.supersonic.chat.core.parser.sql.rule; +package com.tencent.supersonic.headless.core.chat.parser.rule; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MAX; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MIN; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.TOPN; - -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.util.AbstractMap; import java.util.HashMap; @@ -23,9 +17,14 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MAX; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MIN; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.TOPN; /** * AggregateTypeParser extracts aggregation type specified in the user query diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/ContextInheritParser.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/ContextInheritParser.java index cf6c1e824..b002e8b4d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/ContextInheritParser.java @@ -1,16 +1,16 @@ -package com.tencent.supersonic.chat.core.parser.sql.rule; +package com.tencent.supersonic.headless.core.chat.parser.rule; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricModelQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricSemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricModelQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricTagQuery; import lombok.extern.slf4j.Slf4j; import java.util.AbstractMap; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/RuleSqlParser.java similarity index 70% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/RuleSqlParser.java index b176e2ca9..ddee89d04 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/RuleSqlParser.java @@ -1,14 +1,14 @@ -package com.tencent.supersonic.chat.core.parser.sql.rule; +package com.tencent.supersonic.headless.core.chat.parser.rule; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; +import lombok.extern.slf4j.Slf4j; import java.util.Arrays; import java.util.List; -import lombok.extern.slf4j.Slf4j; /** * RuleSqlParser resolves a specific SemanticQuery according to co-appearance @@ -20,8 +20,7 @@ public class RuleSqlParser implements SemanticParser { private static List auxiliaryParsers = Arrays.asList( new ContextInheritParser(), new TimeRangeParser(), - new AggregateTypeParser(), - new AgentCheckParser() + new AggregateTypeParser() ); @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/TimeRangeParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/TimeRangeParser.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/TimeRangeParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/TimeRangeParser.java index fb4a5c6d5..2adb57613 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/TimeRangeParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/rule/TimeRangeParser.java @@ -1,29 +1,28 @@ -package com.tencent.supersonic.chat.core.parser.sql.rule; +package com.tencent.supersonic.headless.core.chat.parser.rule; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; +import com.xkzhangsan.time.nlp.TimeNLP; +import com.xkzhangsan.time.nlp.TimeNLPUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.Strings; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.LocalDate; -import java.util.Stack; import java.util.Date; import java.util.List; +import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.xkzhangsan.time.nlp.TimeNLP; -import com.xkzhangsan.time.nlp.TimeNLPUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; - /** * TimeRangeParser extracts time range specified in the user query * based on keyword matching. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/BaseSemanticQuery.java similarity index 60% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/BaseSemanticQuery.java index f8288bb3f..2d883d490 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/BaseSemanticQuery.java @@ -1,33 +1,24 @@ -package com.tencent.supersonic.chat.core.query; +package com.tencent.supersonic.headless.core.chat.query; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import java.io.Serializable; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; @Slf4j @@ -36,39 +27,6 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { protected SemanticParseInfo parseInfo = new SemanticParseInfo(); - protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - - @Override - public String explain(User user) { - ExplainSqlReq explainSqlReq = null; - try { - SqlInfo sqlInfo = parseInfo.getSqlInfo(); - - if (StringUtils.isNotBlank(sqlInfo.getCorrectS2SQL())) { - //sql - explainSqlReq = ExplainSqlReq.builder() - .queryTypeEnum(QueryType.SQL) - .queryReq(QueryReqBuilder.buildS2SQLReq( - sqlInfo.getCorrectS2SQL(), parseInfo.getDataSetId() - )) - .build(); - } else { - //struct - explainSqlReq = ExplainSqlReq.builder() - .queryTypeEnum(QueryType.STRUCT) - .queryReq(QueryReqBuilder.buildStructReq(parseInfo)) - .build(); - } - ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user); - if (Objects.nonNull(explain)) { - return explain.getSql(); - } - } catch (Exception e) { - log.error("explain error explainSqlReq:{}", explainSqlReq, e); - } - return null; - } - @Override public SemanticParseInfo getParseInfo() { return parseInfo; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/QueryManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/QueryManager.java similarity index 69% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/QueryManager.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/QueryManager.java index 231f3bbf9..19fa5b069 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/QueryManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/QueryManager.java @@ -1,10 +1,9 @@ -package com.tencent.supersonic.chat.core.query; +package com.tencent.supersonic.headless.core.chat.query; -import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; -import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricSemanticQuery; -import com.tencent.supersonic.chat.core.query.rule.tag.TagSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.LLMSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.tag.TagSemanticQuery; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -14,14 +13,11 @@ import java.util.concurrent.ConcurrentHashMap; public class QueryManager { private static Map ruleQueryMap = new ConcurrentHashMap<>(); - private static Map pluginQueryMap = new ConcurrentHashMap<>(); private static Map llmQueryMap = new ConcurrentHashMap<>(); public static void register(SemanticQuery query) { if (query instanceof RuleSemanticQuery) { ruleQueryMap.put(query.getQueryMode(), (RuleSemanticQuery) query); - } else if (query instanceof PluginSemanticQuery) { - pluginQueryMap.put(query.getQueryMode(), (PluginSemanticQuery) query); } else if (query instanceof LLMSemanticQuery) { llmQueryMap.put(query.getQueryMode(), (LLMSemanticQuery) query); } @@ -31,9 +27,6 @@ public class QueryManager { if (containsRuleQuery(queryMode)) { return createRuleQuery(queryMode); } - if (containsPluginQuery(queryMode)) { - return createPluginQuery(queryMode); - } return createLLMQuery(queryMode); } @@ -43,11 +36,6 @@ public class QueryManager { return (RuleSemanticQuery) getSemanticQuery(queryMode, semanticQuery); } - public static PluginSemanticQuery createPluginQuery(String queryMode) { - PluginSemanticQuery semanticQuery = pluginQueryMap.get(queryMode); - return (PluginSemanticQuery) getSemanticQuery(queryMode, semanticQuery); - } - public static LLMSemanticQuery createLLMQuery(String queryMode) { LLMSemanticQuery semanticQuery = llmQueryMap.get(queryMode); return (LLMSemanticQuery) getSemanticQuery(queryMode, semanticQuery); @@ -85,10 +73,6 @@ public class QueryManager { return ruleQueryMap.get(queryMode) instanceof TagSemanticQuery; } - public static boolean containsPluginQuery(String queryMode) { - return queryMode != null && pluginQueryMap.containsKey(queryMode); - } - public static RuleSemanticQuery getRuleQuery(String queryMode) { if (queryMode == null) { return null; @@ -100,8 +84,4 @@ public class QueryManager { return new ArrayList<>(ruleQueryMap.values()); } - public static List getPluginQueryModes() { - return new ArrayList<>(pluginQueryMap.keySet()); - } - } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/SemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/SemanticQuery.java similarity index 57% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/SemanticQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/SemanticQuery.java index fcd4aeed3..3fb03c28e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/SemanticQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/SemanticQuery.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.core.query; +package com.tencent.supersonic.headless.core.chat.query; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import org.apache.calcite.sql.parser.SqlParseException; /** @@ -13,12 +13,10 @@ public interface SemanticQuery { String getQueryMode(); - QueryResult execute(User user) throws SqlParseException; + SemanticQueryReq buildSemanticQueryReq() throws SqlParseException; void initS2Sql(SemanticSchema semanticSchema, User user); - String explain(User user); - SemanticParseInfo getParseInfo(); void setParseInfo(SemanticParseInfo parseInfo); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/LLMSemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/LLMSemanticQuery.java new file mode 100644 index 000000000..9a839d9c4 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/LLMSemanticQuery.java @@ -0,0 +1,8 @@ +package com.tencent.supersonic.headless.core.chat.query.llm; + +import com.tencent.supersonic.headless.core.chat.query.BaseSemanticQuery; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class LLMSemanticQuery extends BaseSemanticQuery { +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMReq.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMReq.java index 5dfe3c2ea..525b306e4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.query.llm.s2sql; +package com.tencent.supersonic.headless.core.chat.query.llm.s2sql; import com.fasterxml.jackson.annotation.JsonValue; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMResp.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMResp.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMResp.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMResp.java index 6488cb8a4..d8d6aafaa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMResp.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMResp.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.chat.core.query.llm.s2sql; +package com.tencent.supersonic.headless.core.chat.query.llm.s2sql; + +import lombok.Data; import java.util.List; import java.util.Map; -import lombok.Data; @Data public class LLMResp { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlQuery.java new file mode 100644 index 000000000..7d1b09883 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlQuery.java @@ -0,0 +1,40 @@ +package com.tencent.supersonic.headless.core.chat.query.llm.s2sql; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.llm.LLMSemanticQuery; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LLMSqlQuery extends LLMSemanticQuery { + + public static final String QUERY_MODE = "LLM_S2SQL"; + + public LLMSqlQuery() { + QueryManager.register(this); + } + + @Override + public String getQueryMode() { + return QUERY_MODE; + } + + @Override + public SemanticQueryReq buildSemanticQueryReq() { + + String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); + return QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId()); + } + + @Override + public void initS2Sql(SemanticSchema semanticSchema, User user) { + SqlInfo sqlInfo = parseInfo.getSqlInfo(); + sqlInfo.setCorrectS2SQL(sqlInfo.getS2SQL()); + } +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlResp.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlResp.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlResp.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlResp.java index 6883bad73..c84f3702a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlResp.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/llm/s2sql/LLMSqlResp.java @@ -1,13 +1,13 @@ -package com.tencent.supersonic.chat.core.query.llm.s2sql; +package com.tencent.supersonic.headless.core.chat.query.llm.s2sql; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; import java.util.Map; -import lombok.Data; -import lombok.Builder; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; - @Data @Builder diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatchOption.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatchOption.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatchOption.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatchOption.java index 080868b51..bbcbd40fd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatchOption.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatchOption.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.query.rule; +package com.tencent.supersonic.headless.core.chat.query.rule; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatcher.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatcher.java index 5f4dbadfa..2bc9482da 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/QueryMatcher.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.chat.core.query.rule; +package com.tencent.supersonic.headless.core.chat.query.rule; + -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import lombok.Data; import lombok.ToString; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/RuleSemanticQuery.java similarity index 75% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/RuleSemanticQuery.java index f4fbf00b7..fcc64889d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/RuleSemanticQuery.java @@ -1,53 +1,47 @@ -package com.tencent.supersonic.chat.core.query.rule; +package com.tencent.supersonic.headless.core.chat.query.rule; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.BaseSemanticQuery; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.core.chat.query.BaseSemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; @Slf4j @ToString public abstract class RuleSemanticQuery extends BaseSemanticQuery { protected QueryMatcher queryMatcher = new QueryMatcher(); - protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); public RuleSemanticQuery() { QueryManager.register(this); } public List match(List candidateElementMatches, - QueryContext queryCtx) { + QueryContext queryCtx) { return queryMatcher.match(candidateElementMatches); } @@ -140,7 +134,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { private void addToFilters(Map> id2Values, SemanticParseInfo parseInfo, SemanticSchema semanticSchema, SchemaElementType entity) { - if (Objects.isNull(id2Values) || id2Values.isEmpty()) { + if (id2Values == null || id2Values.isEmpty()) { return; } for (Entry> entry : id2Values.entrySet()) { @@ -184,7 +178,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { } @Override - public QueryResult execute(User user) { + public SemanticQueryReq buildSemanticQueryReq() { String queryMode = parseInfo.getQueryMode(); if (parseInfo.getDataSetId() == null || StringUtils.isEmpty(queryMode) @@ -194,33 +188,15 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { throw new RuntimeException("not find QueryMode"); } - QueryResult queryResult = new QueryResult(); QueryStructReq queryStructReq = convertQueryStruct(); - - SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); - - if (queryResp != null) { - queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); - } - - String sql = queryResp == null ? null : queryResp.getSql(); - List> resultList = queryResp == null ? new ArrayList<>() - : queryResp.getResultList(); - List columns = queryResp == null ? new ArrayList<>() : queryResp.getColumns(); - queryResult.setQuerySql(sql); - queryResult.setQueryResults(resultList); - queryResult.setQueryColumns(columns); - queryResult.setQueryMode(queryMode); - queryResult.setQueryState(QueryState.SUCCESS); - - return queryResult; + return queryStructReq.convert(true); } protected boolean isMultiStructQuery() { return false; } - public QueryResult multiStructExecute(User user) { + public SemanticQueryReq multiStructExecute() { String queryMode = parseInfo.getQueryMode(); if (parseInfo.getDataSetId() != null || StringUtils.isEmpty(queryMode) @@ -230,23 +206,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { throw new RuntimeException("not find QueryMode"); } - QueryResult queryResult = new QueryResult(); - QueryMultiStructReq queryMultiStructReq = convertQueryMultiStruct(); - SemanticQueryResp queryResp = semanticInterpreter.queryByMultiStruct(queryMultiStructReq, user); - if (queryResp != null) { - queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); - } - String sql = queryResp == null ? null : queryResp.getSql(); - List> resultList = queryResp == null ? new ArrayList<>() - : queryResp.getResultList(); - List columns = queryResp == null ? new ArrayList<>() : queryResp.getColumns(); - queryResult.setQuerySql(sql); - queryResult.setQueryResults(resultList); - queryResult.setQueryColumns(columns); - queryResult.setQueryMode(queryMode); - queryResult.setQueryState(QueryState.SUCCESS); - - return queryResult; + return convertQueryMultiStruct(); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricFilterQuery.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricFilterQuery.java index f5ccf41c4..135deb9df 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricFilterQuery.java @@ -1,23 +1,24 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; +package com.tencent.supersonic.headless.core.chat.query.rule.metric; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.common.pojo.enums.FilterType; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.FilterType; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; + +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; + @Slf4j @Component @@ -36,11 +37,11 @@ public class MetricFilterQuery extends MetricSemanticQuery { } @Override - public QueryResult execute(User user) { + public SemanticQueryReq buildSemanticQueryReq() { if (!isMultiStructQuery()) { - return super.execute(user); + return super.buildSemanticQueryReq(); } - return super.multiStructExecute(user); + return super.multiStructExecute(); } protected boolean isMultiStructQuery() { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricGroupByQuery.java similarity index 61% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricGroupByQuery.java index 8756a7f14..70380c83f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricGroupByQuery.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; +package com.tencent.supersonic.headless.core.chat.query.rule.metric; import org.springframework.stereotype.Component; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Component public class MetricGroupByQuery extends MetricSemanticQuery { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricModelQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricModelQuery.java new file mode 100644 index 000000000..f937a105d --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricModelQuery.java @@ -0,0 +1,23 @@ +package com.tencent.supersonic.headless.core.chat.query.rule.metric; + +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import org.springframework.stereotype.Component; + +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; +@Component +public class MetricModelQuery extends MetricSemanticQuery { + + public static final String QUERY_MODE = "METRIC_MODEL"; + + public MetricModelQuery() { + super(); + queryMatcher.addOption(SchemaElementType.DATASET, OPTIONAL, AT_MOST, 1); + } + + @Override + public String getQueryMode() { + return QUERY_MODE; + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricSemanticQuery.java similarity index 79% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricSemanticQuery.java index ff66a1daf..e2fa6eee9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricSemanticQuery.java @@ -1,23 +1,23 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; +package com.tencent.supersonic.headless.core.chat.query.rule.metric; + -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; import java.time.LocalDate; import java.util.List; import java.util.Objects; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC; - +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTagQuery.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTagQuery.java index b26d7765a..68e4c39fe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTagQuery.java @@ -1,25 +1,24 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; +package com.tencent.supersonic.headless.core.chat.query.rule.metric; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterType; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j @Component public class MetricTagQuery extends MetricSemanticQuery { @@ -38,11 +37,11 @@ public class MetricTagQuery extends MetricSemanticQuery { } @Override - public QueryResult execute(User user) { + public SemanticQueryReq buildSemanticQueryReq() { if (!isMultiStructQuery()) { - return super.execute(user); + return super.buildSemanticQueryReq(); } - return super.multiStructExecute(user); + return super.multiStructExecute(); } protected boolean isMultiStructQuery() { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTopNQuery.java similarity index 75% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTopNQuery.java index ff8df095a..1bbf35f6e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/metric/MetricTopNQuery.java @@ -1,23 +1,24 @@ -package com.tencent.supersonic.chat.core.query.rule.metric; +package com.tencent.supersonic.headless.core.chat.query.rule.metric; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; - -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.springframework.stereotype.Component; + +import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Component public class MetricTopNQuery extends MetricSemanticQuery { @@ -35,7 +36,7 @@ public class MetricTopNQuery extends MetricSemanticQuery { @Override public List match(List candidateElementMatches, - QueryContext queryCtx) { + QueryContext queryCtx) { Matcher matcher = INTENT_PATTERN.matcher(queryCtx.getQueryText()); if (matcher.matches()) { return super.match(candidateElementMatches, queryCtx); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagDetailQuery.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagDetailQuery.java index fc4565d59..b18fb9e92 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagDetailQuery.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.core.query.rule.tag; - -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG; +package com.tencent.supersonic.headless.core.chat.query.rule.tag; import org.springframework.stereotype.Component; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; + @Component public class TagDetailQuery extends TagSemanticQuery { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagFilterQuery.java similarity index 62% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagFilterQuery.java index fe42f766a..f2c3344e9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagFilterQuery.java @@ -1,14 +1,13 @@ -package com.tencent.supersonic.chat.core.query.rule.tag; - -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG_VALUE; +package com.tencent.supersonic.headless.core.chat.query.rule.tag; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG_VALUE; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j @Component diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagIdQuery.java similarity index 60% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagIdQuery.java index d4c83e512..8e3fa1438 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagIdQuery.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.chat.core.query.rule.tag; - -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +package com.tencent.supersonic.headless.core.chat.query.rule.tag; import org.springframework.stereotype.Component; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; + @Component public class TagIdQuery extends TagListQuery { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagListQuery.java similarity index 90% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagListQuery.java index 1e0e189b9..532f98801 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagListQuery.java @@ -1,14 +1,14 @@ -package com.tencent.supersonic.chat.core.query.rule.tag; +package com.tencent.supersonic.headless.core.chat.query.rule.tag; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import org.apache.commons.collections.CollectionUtils; import java.util.LinkedHashSet; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagSemanticQuery.java similarity index 77% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagSemanticQuery.java index 9f881a859..1f855ee05 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/query/rule/tag/TagSemanticQuery.java @@ -1,14 +1,14 @@ -package com.tencent.supersonic.chat.core.query.rule.tag; +package com.tencent.supersonic.headless.core.chat.query.rule.tag; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; import java.time.LocalDate; @@ -16,8 +16,8 @@ import java.util.List; import java.util.Objects; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.headless.core.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j public abstract class TagSemanticQuery extends RuleSemanticQuery { @@ -31,7 +31,7 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery { @Override public List match(List candidateElementMatches, - QueryContext queryCtx) { + QueryContext queryCtx) { return super.match(candidateElementMatches, queryCtx); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/AggregatorConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/AggregatorConfig.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java index 919b10370..e779b3db9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/AggregatorConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/AggregatorConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ChatLocalFileConfig.java similarity index 95% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ChatLocalFileConfig.java index 14023a691..b2f47ed16 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ChatLocalFileConfig.java @@ -1,14 +1,13 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; - -import java.io.FileNotFoundException; - import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; +import java.io.FileNotFoundException; + @Data @Configuration diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetric.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetric.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java index e5acb8354..408af65c9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetric.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetric.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetricInfo.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java similarity index 89% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetricInfo.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java index deecf4db4..3c7db1fe4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultMetricInfo.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultMetricInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import com.tencent.supersonic.common.pojo.Constants; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultSemanticConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultSemanticConfig.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java index caf665e6e..c07552b92 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/DefaultSemanticConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/DefaultSemanticConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/Dim4Dict.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/Dim4Dict.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java index bfe7d7d36..ee35b5a60 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/Dim4Dict.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/Dim4Dict.java @@ -1,11 +1,12 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; +import java.util.List; + @Data @AllArgsConstructor @ToString diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityDetailData.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java similarity index 80% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityDetailData.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java index 9669f914b..d485cf2fb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityDetailData.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java @@ -1,7 +1,8 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; + +import lombok.Data; import java.util.List; -import lombok.Data; /** * when query an entity, return related dimension/metric info diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityInternalDetail.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityInternalDetail.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java index 7b25c269f..ce9d4dceb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/EntityInternalDetail.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java @@ -1,11 +1,12 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class EntityInternalDetail { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LLMParserConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/LLMParserConfig.java similarity index 91% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LLMParserConfig.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/LLMParserConfig.java index 9b078ce8b..dc077799d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LLMParserConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/LLMParserConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java index 0059a8eb6..5871d8fc6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/config/OptimizationConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.chat.core.config; +package com.tencent.supersonic.headless.core.config; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; import com.tencent.supersonic.common.service.SysParameterService; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.SqlGenerationMode; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/KnowledgeService.java similarity index 66% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/KnowledgeService.java index 050c7d821..47881f11c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/KnowledgeService.java @@ -1,17 +1,11 @@ -package com.tencent.supersonic.headless.server.service.impl; +package com.tencent.supersonic.headless.core.knowledge; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; -import com.tencent.supersonic.headless.server.service.KnowledgeService; -import com.tencent.supersonic.headless.server.service.DataSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -19,15 +13,8 @@ import java.util.stream.Collectors; @Service @Slf4j -public class KnowledgeServiceImpl implements KnowledgeService { +public class KnowledgeService { - private final DataSetService dataSetService; - - public KnowledgeServiceImpl(DataSetService dataSetService) { - this.dataSetService = dataSetService; - } - - @Override public void updateSemanticKnowledge(List natures) { List prefixes = natures.stream() @@ -45,7 +32,6 @@ public class KnowledgeServiceImpl implements KnowledgeService { SearchService.loadSuffix(suffixes); } - @Override public void reloadAllData(List natures) { // 1. reload custom knowledge try { @@ -58,7 +44,6 @@ public class KnowledgeServiceImpl implements KnowledgeService { updateOnlineKnowledge(natures); } - @Override public void updateOnlineKnowledge(List natures) { try { updateSemanticKnowledge(natures); @@ -67,15 +52,11 @@ public class KnowledgeServiceImpl implements KnowledgeService { } } - @Override - public List getTerms(String text) { - Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>()); + public List getTerms(String text, Map> modelIdToDataSetIds) { return HanlpHelper.getTerms(text, modelIdToDataSetIds); } - @Override - public List prefixSearch(String key, int limit, Set dataSetIds) { - Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds)); + public List prefixSearch(String key, int limit, Map> modelIdToDataSetIds) { return prefixSearchByModel(key, limit, modelIdToDataSetIds); } @@ -84,9 +65,7 @@ public class KnowledgeServiceImpl implements KnowledgeService { return SearchService.prefixSearch(key, limit, modelIdToDataSetIds); } - @Override - public List suffixSearch(String key, int limit, Set dataSetIds) { - Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds)); + public List suffixSearch(String key, int limit, Map> modelIdToDataSetIds) { return suffixSearchByModel(key, limit, modelIdToDataSetIds.keySet()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MetaEmbeddingService.java similarity index 87% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MetaEmbeddingService.java index 24f0e7b32..7ea3f541a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MetaEmbeddingService.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.service.impl; +package com.tencent.supersonic.headless.core.knowledge; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.Constants; @@ -7,8 +7,6 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.MetaEmbeddingService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -26,19 +24,15 @@ import java.util.stream.Collectors; @Service @Slf4j -public class MetaEmbeddingServiceImpl implements MetaEmbeddingService { +public class MetaEmbeddingService { private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore(); @Autowired private EmbeddingConfig embeddingConfig; - @Autowired - private DataSetService dataSetService; - - @Override - public List retrieveQuery(List dataSetIds, RetrieveQuery retrieveQuery, int num) { + public List retrieveQuery(RetrieveQuery retrieveQuery, int num, + Map> modelIdToDataSetIds) { // dataSetIds->modelIds - Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(dataSetIds); Set allModels = modelIdToDataSetIds.keySet(); if (CollectionUtils.isNotEmpty(allModels) && allModels.size() == 1) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java index 656e8dc48..155ee18d9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java @@ -7,9 +7,9 @@ import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/ChatContext.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ChatContext.java similarity index 59% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/ChatContext.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ChatContext.java index 122dceb62..f9cde618b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/ChatContext.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ChatContext.java @@ -1,13 +1,13 @@ -package com.tencent.supersonic.chat.core.pojo; +package com.tencent.supersonic.headless.core.pojo; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; + +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import lombok.Data; @Data public class ChatContext { private Integer chatId; - private Integer agentId; private String queryText; private SemanticParseInfo parseInfo = new SemanticParseInfo(); private String user; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryContext.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryContext.java index d8bd28c53..5236d65aa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryContext.java @@ -1,28 +1,27 @@ -package com.tencent.supersonic.chat.core.pojo; +package com.tencent.supersonic.headless.core.pojo; import com.fasterxml.jackson.annotation.JsonIgnore; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.core.config.OptimizationConfig; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + @Data @Builder @NoArgsConstructor @@ -31,23 +30,16 @@ public class QueryContext { private String queryText; private Integer chatId; - private Long dataSetId; + private Set dataSetIds; + private Map> modelIdToDataSetIds; private User user; - private boolean saveAnswer = true; - private Integer agentId; + private boolean saveAnswer; + private boolean enableLLM; private QueryFilters queryFilters; private List candidateQueries = new ArrayList<>(); private SchemaMapInfo mapInfo = new SchemaMapInfo(); @JsonIgnore private SemanticSchema semanticSchema; - @JsonIgnore - private Agent agent; - @JsonIgnore - private Map modelIdToChatRichConfig; - @JsonIgnore - private Map nameToPlugin; - @JsonIgnore - private List pluginList; public List getCandidateQueries() { OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 1a55dbc53..618ced7e3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -1,26 +1,35 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.chat.parser.JavaLLMProxy; +import com.tencent.supersonic.headless.core.chat.parser.LLMProxy; +import com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver; import com.tencent.supersonic.headless.core.executor.JdbcExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter; -import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer; -import com.tencent.supersonic.headless.core.planner.QueryOptimizer; -import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser; import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter; import com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter; +import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter; +import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter; +import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer; +import com.tencent.supersonic.headless.core.planner.QueryOptimizer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.io.support.SpringFactoriesLoader; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** * HeadlessConverter QueryOptimizer QueryExecutor object factory */ +@Slf4j public class ComponentFactory { private static List headlessConverters = new ArrayList<>(); @@ -28,6 +37,9 @@ public class ComponentFactory { private static List queryExecutors = new ArrayList<>(); private static SqlParser sqlParser; + private static LLMProxy llmProxy; + private static DataSetResolver modelResolver; + static { initSemanticConverter(); initQueryOptimizer(); @@ -89,4 +101,41 @@ public class ComponentFactory { headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); } + public static LLMProxy getLLMProxy() { + //1.Preferentially retrieve from environment variables + String llmProxyEnv = System.getenv("llmProxy"); + if (StringUtils.isNotBlank(llmProxyEnv)) { + Map implementations = ContextUtils.getBeansOfType(LLMProxy.class); + llmProxy = implementations.entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(llmProxyEnv)) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + } + //2.default JavaLLMProxy + if (Objects.isNull(llmProxy)) { + llmProxy = ContextUtils.getBean(JavaLLMProxy.class); + } + log.info("llmProxy:{}", llmProxy); + return llmProxy; + } + + public static DataSetResolver getModelResolver() { + if (Objects.isNull(modelResolver)) { + modelResolver = init(DataSetResolver.class); + } + return modelResolver; + } + + private static List init(Class factoryType, List list) { + list.addAll(SpringFactoriesLoader.loadFactories(factoryType, + Thread.currentThread().getContextClassLoader())); + return list; + } + + private static T init(Class factoryType) { + return SpringFactoriesLoader.loadFactories(factoryType, + Thread.currentThread().getContextClassLoader()).get(0); + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryReqBuilder.java similarity index 97% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryReqBuilder.java index dd5435d1c..3f3e5cfe9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryReqBuilder.java @@ -1,22 +1,22 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.utils; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/S2SqlDateHelper.java similarity index 94% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/S2SqlDateHelper.java index 0ea41ef2b..ce646c0b5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/S2SqlDateHelper.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.utils; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.util.DatePeriodEnum; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import org.apache.commons.lang3.tuple.Pair; import java.util.Objects; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java index 97380b07e..c14db70c8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SchemaMatchHelper.java @@ -1,12 +1,13 @@ -package com.tencent.supersonic.chat.core.utils; +package com.tencent.supersonic.headless.core.utils; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /** * Schema Match Helper @@ -20,8 +21,8 @@ public class SchemaMatchHelper { } Set metricDimensionDetectWordSet = matches.stream() - .filter(elementMatch -> isMetricOrDimension(elementMatch)) - .map(entry -> entry.getDetectWord()).collect(Collectors.toSet()); + .filter(SchemaMatchHelper::isMetricOrDimension) + .map(SchemaElementMatch::getDetectWord).collect(Collectors.toSet()); matches.removeIf(elementMatch -> { if (!isMetricOrDimension(elementMatch)) { diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java similarity index 89% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java index 045323cf3..f8cbb7e35 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrectorTest.java @@ -1,24 +1,26 @@ package com.tencent.supersonic.chat.core.corrector; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.core.chat.corrector.SchemaCorrector; +import com.tencent.supersonic.headless.core.chat.parser.llm.ParseResult; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq; +import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.junit.jupiter.api.Test; + +import static org.testng.Assert.assertEquals; class SchemaCorrectorTest { @@ -92,8 +94,8 @@ class SchemaCorrectorTest { parseResult = objectMapper.readValue(json, ParseResult.class); - List linkingValues = new ArrayList<>(); - ElementValue elementValue = new ElementValue(); + List linkingValues = new ArrayList<>(); + LLMReq.ElementValue elementValue = new LLMReq.ElementValue(); elementValue.setFieldName("商务组"); elementValue.setFieldValue("xxx"); linkingValues.add(elementValue); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java similarity index 93% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java index 11fcb5250..d85f89223 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import org.junit.Assert; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.core.chat.corrector.TimeCorrector; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import org.junit.jupiter.api.Test; +import org.testng.Assert; class TimeCorrectorTest { diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java similarity index 75% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java index 6113ba85e..9f14540c4 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.mapper; import com.hankcs.hanlp.algorithm.EditDistance; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; class LoadRemoveServiceTest { @@ -10,6 +10,6 @@ class LoadRemoveServiceTest { @Test void edit() { int compute = EditDistance.compute("在", "在你的身边"); - Assert.assertEquals(compute, 4); + Assertions.assertEquals(compute, 4); } } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java similarity index 93% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java index 1c0e4ed3f..2c850beea 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.chat.core.parser.aggregate; - -import com.tencent.supersonic.chat.core.parser.sql.rule.AggregateTypeParser; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; -import org.junit.Assert; +import com.tencent.supersonic.headless.core.chat.parser.rule.AggregateTypeParser; import org.junit.jupiter.api.Test; +import org.testng.Assert; class AggregateTypeParserTest { diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java similarity index 89% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java index 33b3cd9c8..8565427f3 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.chat.core.s2sql; -import com.tencent.supersonic.chat.core.parser.sql.llm.LLMResponseService; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.parser.llm.LLMResponseService; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlResp; +import org.junit.jupiter.api.Test; +import org.testng.Assert; + import java.util.HashMap; import java.util.Map; -import org.junit.Assert; -import org.junit.jupiter.api.Test; class LLMResponseServiceTest { diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java similarity index 96% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java index 1782eae44..9f47b258c 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.s2sql; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import org.junit.jupiter.api.Test; import org.mockito.Mockito; diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java similarity index 95% rename from chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java rename to headless/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java index b68392229..5533d5b78 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelperTest.java @@ -1,19 +1,21 @@ package com.tencent.supersonic.chat.core.utils; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.utils.S2SqlDateHelper; import org.apache.commons.lang3.tuple.Pair; import org.junit.Assert; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.List; diff --git a/chat/core/src/test/resources/application.yaml b/headless/core/src/test/resources/application.yaml similarity index 100% rename from chat/core/src/test/resources/application.yaml rename to headless/core/src/test/resources/application.yaml diff --git a/chat/core/src/test/resources/data/README.url b/headless/core/src/test/resources/data/README.url similarity index 100% rename from chat/core/src/test/resources/data/README.url rename to headless/core/src/test/resources/data/README.url diff --git a/chat/core/src/test/resources/data/dictionary/CoreNatureDictionary.mini.txt b/headless/core/src/test/resources/data/dictionary/CoreNatureDictionary.mini.txt similarity index 100% rename from chat/core/src/test/resources/data/dictionary/CoreNatureDictionary.mini.txt rename to headless/core/src/test/resources/data/dictionary/CoreNatureDictionary.mini.txt diff --git a/chat/core/src/test/resources/data/dictionary/CoreNatureDictionary.ngram.mini.txt b/headless/core/src/test/resources/data/dictionary/CoreNatureDictionary.ngram.mini.txt similarity index 100% rename from chat/core/src/test/resources/data/dictionary/CoreNatureDictionary.ngram.mini.txt rename to headless/core/src/test/resources/data/dictionary/CoreNatureDictionary.ngram.mini.txt diff --git a/chat/core/src/test/resources/data/dictionary/custom/DimValue_1_2.txt b/headless/core/src/test/resources/data/dictionary/custom/DimValue_1_2.txt similarity index 100% rename from chat/core/src/test/resources/data/dictionary/custom/DimValue_1_2.txt rename to headless/core/src/test/resources/data/dictionary/custom/DimValue_1_2.txt diff --git a/chat/core/src/test/resources/data/dictionary/other/CharTable.txt b/headless/core/src/test/resources/data/dictionary/other/CharTable.txt similarity index 100% rename from chat/core/src/test/resources/data/dictionary/other/CharTable.txt rename to headless/core/src/test/resources/data/dictionary/other/CharTable.txt diff --git a/chat/core/src/test/resources/data/dictionary/other/TagPKU98.csv b/headless/core/src/test/resources/data/dictionary/other/TagPKU98.csv similarity index 100% rename from chat/core/src/test/resources/data/dictionary/other/TagPKU98.csv rename to headless/core/src/test/resources/data/dictionary/other/TagPKU98.csv diff --git a/chat/core/src/test/resources/data/version.txt b/headless/core/src/test/resources/data/version.txt similarity index 100% rename from chat/core/src/test/resources/data/version.txt rename to headless/core/src/test/resources/data/version.txt diff --git a/chat/core/src/test/resources/db/chat-data-h2.sql b/headless/core/src/test/resources/db/chat-data-h2.sql similarity index 100% rename from chat/core/src/test/resources/db/chat-data-h2.sql rename to headless/core/src/test/resources/db/chat-data-h2.sql diff --git a/chat/core/src/test/resources/db/chat-schema-h2.sql b/headless/core/src/test/resources/db/chat-schema-h2.sql similarity index 100% rename from chat/core/src/test/resources/db/chat-schema-h2.sql rename to headless/core/src/test/resources/db/chat-schema-h2.sql diff --git a/chat/core/src/test/resources/hanlp.properties b/headless/core/src/test/resources/hanlp.properties similarity index 100% rename from chat/core/src/test/resources/hanlp.properties rename to headless/core/src/test/resources/hanlp.properties diff --git a/headless/server/pom.xml b/headless/server/pom.xml index b5950ff54..bd7c2578a 100644 --- a/headless/server/pom.xml +++ b/headless/server/pom.xml @@ -78,6 +78,12 @@ junit-jupiter test + + org.mockito + mockito-inline + ${mockito-inline.version} + test + org.springframework.boot spring-boot-starter-test diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 86be574a1..be3a70701 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -47,7 +47,7 @@ public class DimValueAspect { @Autowired private DimensionService dimensionService; - @Around("execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByReq(..))") + @Around("execution(* com.tencent.supersonic.headless.server.service.ChatQueryService.queryByReq(..))") public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapEnable) { log.debug("dimensionValueMapEnable is false, skip dimensionValueMap"); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java similarity index 86% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java index 217d4ec9e..6fe0a1325 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java @@ -1,13 +1,9 @@ -package com.tencent.supersonic.chat.server.listener; +package com.tencent.supersonic.headless.server.listener; + import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.chat.server.service.impl.SchemaService; -import com.tencent.supersonic.chat.server.service.impl.WordService; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; +import com.tencent.supersonic.headless.server.service.impl.WordService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -16,6 +12,9 @@ import org.springframework.core.annotation.Order; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.util.List; +import java.util.concurrent.CompletableFuture; + @Slf4j @Component @Order(2) @@ -25,8 +24,6 @@ public class ApplicationStartedListener implements CommandLineRunner { private KnowledgeService knowledgeService; @Autowired private WordService wordService; - @Autowired - private SchemaService schemaService; @Override public void run(String... args) { @@ -76,8 +73,6 @@ public class ApplicationStartedListener implements CommandLineRunner { log.info("dictWords has changed"); wordService.setPreDictWords(dictWords); knowledgeService.updateOnlineKnowledge(wordService.getAllDictWords()); - schemaService.getCache().refresh(SchemaService.ALL_CACHE); - } catch (Exception e) { log.error("reloadKnowledge error", e); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java similarity index 87% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java index 97cea8c19..74cb5635b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java @@ -1,14 +1,12 @@ -package com.tencent.supersonic.chat.server.listener; +package com.tencent.supersonic.headless.server.listener; -import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; -import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.pojo.enums.EventType; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; @@ -18,16 +16,12 @@ import org.springframework.util.CollectionUtils; @Slf4j public class SchemaDictUpdateListener implements ApplicationListener { - @Autowired - private SchemaService schemaService; - @Async @Override public void onApplicationEvent(DataEvent dataEvent) { if (CollectionUtils.isEmpty(dataEvent.getDataItems())) { return; } - schemaService.getCache().invalidateAll(); dataEvent.getDataItems().forEach(dataItem -> { DictWord dictWord = new DictWord(); dictWord.setWord(dataItem.getName()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatContextDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ChatContextDO.java similarity index 80% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatContextDO.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ChatContextDO.java index 5f5a923c2..eeea9ec5a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/ChatContextDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ChatContextDO.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.chat.server.persistence.dataobject; +package com.tencent.supersonic.headless.server.persistence.dataobject; + +import lombok.Data; import java.io.Serializable; import java.time.Instant; -import lombok.Data; @Data public class ChatContextDO implements Serializable { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/StatisticsDO.java similarity index 90% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/StatisticsDO.java index cd2ee5da4..58c4e529c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/StatisticsDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/StatisticsDO.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.server.persistence.dataobject; +package com.tencent.supersonic.headless.server.persistence.dataobject; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatContextMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ChatContextMapper.java similarity index 61% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatContextMapper.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ChatContextMapper.java index c30b73610..9d73285ce 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/ChatContextMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ChatContextMapper.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.server.persistence.mapper; +package com.tencent.supersonic.headless.server.persistence.mapper; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.ChatContextDO; import org.apache.ibatis.annotations.Mapper; @Mapper diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/StatisticsMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatisticsMapper.java similarity index 60% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/StatisticsMapper.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatisticsMapper.java index 5bd0b15f1..8cfed656e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/StatisticsMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatisticsMapper.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.server.persistence.mapper; +package com.tencent.supersonic.headless.server.persistence.mapper; -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ChatContextRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ChatContextRepository.java new file mode 100644 index 000000000..ea73202e1 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ChatContextRepository.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.headless.server.persistence.repository; + +import com.tencent.supersonic.headless.core.pojo.ChatContext; + +public interface ChatContextRepository { + + ChatContext getOrCreateContext(int chatId); + + void updateContext(ChatContext chatCtx); + +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ChatContextRepositoryImpl.java similarity index 83% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ChatContextRepositoryImpl.java index 5c45f7b74..98c66d3cd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ChatContextRepositoryImpl.java @@ -1,12 +1,12 @@ -package com.tencent.supersonic.chat.server.persistence.repository.impl; +package com.tencent.supersonic.headless.server.persistence.repository.impl; import com.google.gson.Gson; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO; -import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper; -import com.tencent.supersonic.chat.server.persistence.repository.ChatContextRepository; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.server.persistence.dataobject.ChatContextDO; +import com.tencent.supersonic.headless.server.persistence.mapper.ChatContextMapper; +import com.tencent.supersonic.headless.server.persistence.repository.ChatContextRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Primary; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java new file mode 100644 index 000000000..88b427660 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java @@ -0,0 +1,43 @@ +package com.tencent.supersonic.headless.server.processor; + +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.server.service.impl.SemanticService; +import org.springframework.util.CollectionUtils; +import java.util.List; + +/** + * EntityInfoProcessor fills core attributes of an entity so that + * users get to know which entity is parsed out. + */ +public class EntityInfoProcessor implements ResultProcessor { + + @Override + public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { + List selectedParses = parseResp.getSelectedParses(); + if (CollectionUtils.isEmpty(selectedParses)) { + return; + } + selectedParses.forEach(parseInfo -> { + String queryMode = parseInfo.getQueryMode(); + if (QueryManager.containsRuleQuery(queryMode)) { + return; + } + //1. set entity info + DataSetSchema dataSetSchema = + queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, queryContext.getUser()); + if (QueryManager.isTagQuery(queryMode) + || QueryManager.isMetricQuery(queryMode)) { + parseInfo.setEntityInfo(entityInfo); + } + }); + } +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java similarity index 91% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java index 979fa8ec1..8321fd23c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java @@ -1,16 +1,6 @@ -package com.tencent.supersonic.chat.server.processor.parse; +package com.tencent.supersonic.headless.server.processor; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; @@ -18,6 +8,16 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.server.service.impl.SemanticService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -37,7 +37,7 @@ import java.util.stream.Collectors; * users get to know the details. **/ @Slf4j -public class ParseInfoProcessor implements ParseResultProcessor { +public class ParseInfoProcessor implements ResultProcessor { @Override public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { @@ -83,7 +83,7 @@ public class ParseInfoProcessor implements ParseResultProcessor { log.error("set dimensionFilter error :", e); } - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); + SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema(); if (Objects.isNull(semanticSchema)) { return; } @@ -195,7 +195,7 @@ public class ParseInfoProcessor implements ParseResultProcessor { } protected Map getNameToElement(Long dataSetId) { - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); + SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema(); List dimensions = semanticSchema.getDimensions(dataSetId); List metrics = semanticSchema.getMetrics(dataSetId); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRankProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/QueryRankProcessor.java similarity index 81% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRankProcessor.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/QueryRankProcessor.java index 749a36a73..b07df68eb 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/QueryRankProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/QueryRankProcessor.java @@ -1,12 +1,13 @@ -package com.tencent.supersonic.chat.server.processor.parse; +package com.tencent.supersonic.headless.server.processor; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; + +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -19,7 +20,7 @@ import java.util.stream.Collectors; * a heuristic scoring algorithm and then takes topN. **/ @Slf4j -public class QueryRankProcessor implements ParseResultProcessor { +public class QueryRankProcessor implements ResultProcessor { private static final int candidateTopSize = 5; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/RespBuildProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/RespBuildProcessor.java similarity index 56% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/RespBuildProcessor.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/RespBuildProcessor.java index 4bf9cc540..5fbc26f42 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/RespBuildProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/RespBuildProcessor.java @@ -1,28 +1,25 @@ -package com.tencent.supersonic.chat.server.processor.parse; +package com.tencent.supersonic.headless.server.processor; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.server.service.ChatService; -import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; /** * RespBuildProcessor fill response object with parsing results. **/ @Slf4j -public class RespBuildProcessor implements ParseResultProcessor { +public class RespBuildProcessor implements ResultProcessor { @Override public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { parseResp.setChatId(queryContext.getChatId()); parseResp.setQueryText(queryContext.getQueryText()); List candidateQueries = queryContext.getCandidateQueries(); - ChatService chatService = ContextUtils.getBean(ChatService.class); if (candidateQueries.size() > 0) { List candidateParses = candidateQueries.stream() .map(SemanticQuery::getParseInfo).collect(Collectors.toList()); @@ -31,7 +28,6 @@ public class RespBuildProcessor implements ParseResultProcessor { } else { parseResp.setState(ParseResp.ParseState.FAILED); } - chatService.batchAddParse(chatContext, queryContext, parseResp); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java new file mode 100644 index 000000000..35202640f --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ResultProcessor.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.headless.server.processor; + +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; + +/** + * A ParseResultProcessor wraps things up before returning results to users in parse stage. + */ +public interface ResultProcessor { + + void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext); + +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/SqlInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java similarity index 53% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/SqlInfoProcessor.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java index be7580698..6bd0cb714 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/SqlInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java @@ -1,26 +1,34 @@ -package com.tencent.supersonic.chat.server.processor.parse; +package com.tencent.supersonic.headless.server.processor; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; +import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.server.service.QueryService; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * SqlInfoProcessor adds S2SQL to the parsing results so that * technical users could verify SQL by themselves. **/ -public class SqlInfoProcessor implements ParseResultProcessor { +@Slf4j +public class SqlInfoProcessor implements ResultProcessor { private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); @@ -42,17 +50,26 @@ public class SqlInfoProcessor implements ParseResultProcessor { return; } semanticParseInfos.forEach(parseInfo -> { - addSqlInfo(queryContext, parseInfo); + try { + addSqlInfo(queryContext, parseInfo); + } catch (Exception e) { + log.warn("get sql info failed:{}", parseInfo, e); + } }); } - private void addSqlInfo(QueryContext queryContext, SemanticParseInfo parseInfo) { + private void addSqlInfo(QueryContext queryContext, SemanticParseInfo parseInfo) throws Exception { SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); if (Objects.isNull(semanticQuery)) { return; } semanticQuery.setParseInfo(parseInfo); - String explainSql = semanticQuery.explain(queryContext.getUser()); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + QueryService queryService = ContextUtils.getBean(QueryService.class); + ExplainSqlReq explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq) + .queryTypeEnum(QueryMethod.SQL).build(); + ExplainResp explain = queryService.explain(explainSqlReq, queryContext.getUser()); + String explainSql = explain.getSql(); if (StringUtils.isBlank(explainSql)) { return; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/TimeCostProcessor.java similarity index 59% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/TimeCostProcessor.java index 69aa91ae7..c2e2b7b99 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/TimeCostProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/TimeCostProcessor.java @@ -1,15 +1,16 @@ -package com.tencent.supersonic.chat.server.processor.parse; +package com.tencent.supersonic.headless.server.processor; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; + +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; /** * TimeCostProcessor adds time cost of parsing. **/ @Slf4j -public class TimeCostProcessor implements ParseResultProcessor { +public class TimeCostProcessor implements ResultProcessor { @Override public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index 7db475905..512a88195 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; @@ -19,10 +19,6 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.service.DownloadService; import com.tencent.supersonic.headless.server.service.QueryService; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -30,6 +26,11 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; +import java.util.List; + @RestController @RequestMapping("/api/semantic/query") @Slf4j @@ -109,13 +110,13 @@ public class QueryController { User user = UserHolder.findUser(request, response); String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq()); - if (QueryType.SQL.equals(explainSqlReq.getQueryTypeEnum())) { + if (QueryMethod.SQL.equals(explainSqlReq.getQueryTypeEnum())) { ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() .queryReq(JsonUtil.toObject(queryReqJson, QuerySqlReq.class)) .queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build(); return queryService.explain(explainSqlReqNew, user); } - if (QueryType.STRUCT.equals(explainSqlReq.getQueryTypeEnum())) { + if (QueryMethod.STRUCT.equals(explainSqlReq.getQueryTypeEnum())) { ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() .queryReq(JsonUtil.toObject(queryReqJson, QueryStructReq.class)) .queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetricQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetricQueryApiController.java index 3accd03ed..601631003 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetricQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetricQueryApiController.java @@ -6,8 +6,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.QueryService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -15,6 +13,9 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + @RestController @RequestMapping("/api/semantic/query") @Slf4j diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java index a1c5d306e..086de3a84 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java @@ -4,8 +4,6 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.server.service.QueryService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -13,6 +11,9 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + @RestController @RequestMapping("/api/semantic/query") @Slf4j diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java index 5e4d4cb36..f3584262b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java @@ -4,8 +4,6 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.server.service.QueryService; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -13,6 +11,9 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + @RestController @RequestMapping("/api/semantic/query") @Slf4j diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatContextService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatContextService.java new file mode 100644 index 000000000..e2b22153e --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatContextService.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.headless.core.pojo.ChatContext; + +public interface ChatContextService { + + /*** + * get the model from context + * @param chatId + * @return + */ + Long getContextModel(Integer chatId); + + ChatContext getOrCreateContext(int chatId); + + void updateContext(ChatContext chatCtx); + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatQueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatQueryService.java new file mode 100644 index 000000000..fd0f8db83 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ChatQueryService.java @@ -0,0 +1,30 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; + +/*** + * QueryService for query and search + */ +public interface ChatQueryService { + + ParseResp performParsing(QueryReq queryReq); + + QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception; + + SemanticParseInfo queryContext(Integer chatId); + + QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception; + + EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user); + + Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception; +} + diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java index 07ac0043b..3024c7f81 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java @@ -24,9 +24,12 @@ public interface DataSetService { Map> getModelIdToDataSetIds(List dataSetIds); + Map> getModelIdToDataSetIds(); + List getDataSets(User user); List getDataSetsInheritAuth(User user, Long domainId); SemanticQueryReq convert(QueryDataSetReq queryDataSetReq); + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java deleted file mode 100644 index 3e5e92562..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.tencent.supersonic.headless.server.service; - - -import com.tencent.supersonic.headless.api.pojo.response.S2Term; -import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; - -import java.util.List; -import java.util.Set; - -public interface KnowledgeService { - - List getTerms(String text); - - List prefixSearch(String key, int limit, Set dataSetIds); - - List suffixSearch(String key, int limit, Set dataSetIds); - - void updateSemanticKnowledge(List natures); - - void reloadAllData(List natures); - - void updateOnlineKnowledge(List natures); - -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java deleted file mode 100644 index e24fd8bfb..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.headless.server.service; - -import com.tencent.supersonic.common.util.embedding.RetrieveQuery; -import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; -import java.util.List; - -public interface MetaEmbeddingService { - - List retrieveQuery(List dataSetIds, RetrieveQuery retrieveQuery, int num); - -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index e5375676b..ed21023ee 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -11,8 +11,9 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck; -import java.util.List; + import javax.servlet.http.HttpServletRequest; +import java.util.List; public interface QueryService { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 59b94ca77..56463f10c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -3,12 +3,14 @@ package com.tencent.supersonic.headless.server.service; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemResp; @@ -17,8 +19,6 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import java.util.List; import java.util.concurrent.ExecutionException; @@ -27,6 +27,10 @@ public interface SchemaService { List fetchDataSetSchema(DataSetFilterReq filter); + DataSetSchema getDataSetSchema(Long dataSetId); + + List getDataSetSchema(); + List fetchModelSchemaResps(List modelIds); PageInfo queryDimension(PageDimensionReq pageDimensionReq, User user); @@ -39,8 +43,6 @@ public interface SchemaService { List getModelList(User user, AuthType authType, Long domainId); - List getDataSetList(Long domainId); - SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq); List getStatInfo(ItemUseReq itemUseReq) throws ExecutionException; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java new file mode 100644 index 000000000..df8d618aa --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.api.pojo.response.SearchResult; + +import java.util.List; + + +/** + * search service + */ +public interface SearchService { + + List search(QueryReq queryCtx); + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatContextServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatContextServiceImpl.java new file mode 100644 index 000000000..d5c70b545 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatContextServiceImpl.java @@ -0,0 +1,50 @@ +package com.tencent.supersonic.headless.server.service.impl; + +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.server.persistence.repository.ChatContextRepository; +import com.tencent.supersonic.headless.server.service.ChatContextService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Objects; + + +@Slf4j +@Service +public class ChatContextServiceImpl implements ChatContextService { + + private ChatContextRepository chatContextRepository; + + public ChatContextServiceImpl(ChatContextRepository chatContextRepository) { + this.chatContextRepository = chatContextRepository; + } + + @Override + public Long getContextModel(Integer chatId) { + if (Objects.isNull(chatId)) { + return null; + } + ChatContext chatContext = getOrCreateContext(chatId); + if (Objects.isNull(chatContext)) { + return null; + } + SemanticParseInfo originalSemanticParse = chatContext.getParseInfo(); + if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getDataSetId())) { + return originalSemanticParse.getDataSetId(); + } + return null; + } + + @Override + public ChatContext getOrCreateContext(int chatId) { + return chatContextRepository.getOrCreateContext(chatId); + } + + @Override + public void updateContext(ChatContext chatCtx) { + log.debug("save ChatContext {}", chatCtx); + chatContextRepository.updateContext(chatCtx); + } + +} diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java similarity index 72% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java index 5ea5e73bb..278b012a2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java @@ -1,55 +1,6 @@ -package com.tencent.supersonic.chat.server.service.impl; +package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.corrector.SemanticCorrector; -import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.core.knowledge.SearchService; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.mapper.SchemaMapper; -import com.tencent.supersonic.chat.core.parser.SemanticParser; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; -import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; -import com.tencent.supersonic.chat.core.utils.SimilarQueryManager; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.CostType; -import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO; -import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor; -import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor; -import com.tencent.supersonic.chat.server.service.AgentService; -import com.tencent.supersonic.chat.server.service.ChatService; -import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.chat.server.service.PluginService; -import com.tencent.supersonic.chat.server.service.QueryService; -import com.tencent.supersonic.chat.server.service.SemanticService; -import com.tencent.supersonic.chat.server.service.StatisticsService; -import com.tencent.supersonic.chat.server.service.TimeCost; -import com.tencent.supersonic.chat.server.util.ComponentFactory; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; @@ -63,9 +14,49 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.enums.CostType; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector; +import com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.core.chat.query.QueryManager; +import com.tencent.supersonic.headless.core.chat.query.SemanticQuery; +import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; +import com.tencent.supersonic.headless.core.knowledge.SearchService; +import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; +import com.tencent.supersonic.headless.core.pojo.ChatContext; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO; +import com.tencent.supersonic.headless.server.processor.ResultProcessor; +import com.tencent.supersonic.headless.server.service.ChatContextService; +import com.tencent.supersonic.headless.server.service.ChatQueryService; +import com.tencent.supersonic.headless.server.service.DataSetService; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; @@ -79,15 +70,12 @@ import net.sf.jsqlparser.expression.operators.relational.InExpression; import net.sf.jsqlparser.expression.operators.relational.MinorThan; import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.schema.Column; -import org.apache.calcite.sql.parser.SqlParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -100,39 +88,27 @@ import java.util.Set; import java.util.stream.Collectors; @Service -@Component("chatQueryService") -@Primary @Slf4j -public class QueryServiceImpl implements QueryService { +public class ChatQueryServiceImpl implements ChatQueryService { @Autowired - private ChatService chatService; + private SemanticService semanticService; @Autowired - private StatisticsService statisticsService; - @Autowired - private SimilarQueryManager similarQueryManager; - @Autowired - private SchemaService schemaService; - @Autowired - private AgentService agentService; - - @Autowired - private ConfigService configService; - - @Autowired - private PluginService pluginService; - + private ChatContextService chatContextService; @Autowired private KnowledgeService knowledgeService; + @Autowired + private QueryService queryService; + @Autowired + private DataSetService dataSetService; @Value("${time.threshold: 100}") private Integer timeThreshold; private List schemaMappers = ComponentFactory.getSchemaMappers(); private List semanticParsers = ComponentFactory.getSemanticParsers(); - private List parseProcessors = ComponentFactory.getParseProcessors(); - private List executeProcessors = ComponentFactory.getExecuteProcessors(); private List semanticCorrectors = ComponentFactory.getSemanticCorrectors(); + private List resultProcessors = ComponentFactory.getResultProcessors(); @Override public ParseResp performParsing(QueryReq queryReq) { @@ -141,7 +117,7 @@ public class QueryServiceImpl implements QueryService { QueryContext queryCtx = buildQueryContext(queryReq); // in order to support multi-turn conversation, chat context is needed - ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); + ChatContext chatCtx = chatContextService.getOrCreateContext(queryReq.getChatId()); List timeCostDOList = new ArrayList<>(); // 1. mapper @@ -175,54 +151,32 @@ public class QueryServiceImpl implements QueryService { } } - // 4. processor - parseProcessors.forEach(processor -> { - long startTime = System.currentTimeMillis(); + //4. processor + resultProcessors.forEach(processor -> { processor.process(parseResult, queryCtx, chatCtx); - timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) - .interfaceName(processor.getClass().getSimpleName()) - .type(CostType.PROCESSOR.getType()).build()); - log.debug("{} result:{}", processor.getClass().getSimpleName(), JsonUtil.toString(queryCtx)); }); - - if (Objects.nonNull(parseResult.getQueryId()) && timeCostDOList.size() > 0) { - saveTimeCostInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(), - queryReq.getUser().getName(), queryReq.getChatId().longValue()); - } return parseResult; } private QueryContext buildQueryContext(QueryReq queryReq) { - Integer agentId = queryReq.getAgentId(); - Agent agent = agentService.getAgent(agentId); - - SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - Map modelIdToChatRichConfig = configService.getModelIdToChatRichConfig(); - - Map nameToPlugin = pluginService.getNameToPlugin(); - List pluginList = pluginService.getPluginList(); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(); QueryContext queryCtx = QueryContext.builder() .queryFilters(queryReq.getQueryFilters()) .semanticSchema(semanticSchema) .candidateQueries(new ArrayList<>()) .mapInfo(new SchemaMapInfo()) - .agent(agent) - .modelIdToChatRichConfig(modelIdToChatRichConfig) - .nameToPlugin(nameToPlugin) - .pluginList(pluginList) + .modelIdToDataSetIds(modelIdToDataSetIds) .build(); BeanUtils.copyProperties(queryReq, queryCtx); return queryCtx; } @Override - @TimeCost public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { - ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(), queryReq.getParseId()); - ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId()); List timeCostDOList = new ArrayList<>(); - SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + SemanticParseInfo parseInfo = queryReq.getParseInfo(); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); if (semanticQuery == null) { return null; @@ -230,93 +184,61 @@ public class QueryServiceImpl implements QueryService { semanticQuery.setParseInfo(parseInfo); // in order to support multi-turn conversation, chat context is needed - ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); - chatCtx.setAgentId(queryReq.getAgentId()); - Long startTime = System.currentTimeMillis(); - QueryResult queryResult = semanticQuery.execute(queryReq.getUser()); - - if (queryResult != null) { - timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) - .interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build()); - queryResult.setQueryTimeCost(timeCostDOList.get(0).getCost().longValue()); - saveTimeCostInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(), - queryReq.getUser().getName(), queryReq.getChatId().longValue()); - queryResult.setChatContext(parseInfo); - // update chat context after a successful semantic query - if (QueryState.SUCCESS.equals(queryResult.getQueryState())) { - chatCtx.setParseInfo(parseInfo); - chatService.updateContext(chatCtx); - saveSolvedQuery(queryReq, parseInfo, chatQueryDO, queryResult); - } - chatCtx.setQueryText(queryReq.getQueryText()); - chatCtx.setUser(queryReq.getUser().getName()); - for (ExecuteResultProcessor executeResultProcessor : executeProcessors) { - executeResultProcessor.process(queryResult, parseInfo, queryReq); - } - chatService.updateQuery(queryReq.getQueryId(), queryReq.getParseId(), queryResult, chatCtx); - } else { - chatService.deleteChatQuery(queryReq.getQueryId()); + ChatContext chatCtx = chatContextService.getOrCreateContext(queryReq.getChatId()); + long startTime = System.currentTimeMillis(); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + QueryResult queryResult = doExecution(semanticQueryReq, parseInfo, queryReq.getUser()); + timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) + .interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build()); + queryResult.setQueryTimeCost(timeCostDOList.get(0).getCost().longValue()); + queryResult.setChatContext(parseInfo); + // update chat context after a successful semantic query + if (QueryState.SUCCESS.equals(queryResult.getQueryState()) && queryReq.isSaveAnswer()) { + chatCtx.setParseInfo(parseInfo); + chatContextService.updateContext(chatCtx); } + chatCtx.setQueryText(queryReq.getQueryText()); + chatCtx.setUser(queryReq.getUser().getName()); return queryResult; } - /** - * save time cost data - * - * @param timeCostDOList - * @param queryText - * @param queryId - * @param userName - * @param chatId - */ - private void saveTimeCostInfo(List timeCostDOList, - String queryText, Long queryId, - String userName, Long chatId) { - List list = timeCostDOList.stream() - .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList()); - list.forEach(o -> { - o.setQueryText(queryText); - o.setQuestionId(queryId); - o.setUserName(userName); - o.setChatId(chatId); - o.setCreateTime(new java.util.Date()); - }); - if (list.size() > 0) { - log.info("filterStatistics size:{},data:{}", list.size(), JsonUtil.toString(list)); - statisticsService.batchSaveStatistics(list); + private QueryResult doExecution(SemanticQueryReq semanticQueryReq, + SemanticParseInfo parseInfo, User user) throws Exception { + SemanticQueryResp queryResp = queryService.queryByReq(semanticQueryReq, user); + QueryResult queryResult = new QueryResult(); + if (queryResp != null) { + queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); } - } - private void saveSolvedQuery(ExecuteQueryReq queryReq, SemanticParseInfo parseInfo, - ChatQueryDO chatQueryDO, QueryResult queryResult) { - if (queryResult.getResponse() == null && CollectionUtils.isEmpty(queryResult.getQueryResults())) { - return; - } - similarQueryManager.saveSimilarQuery(SimilarQueryReq.builder().parseId(queryReq.getParseId()) - .queryId(queryReq.getQueryId()) - .agentId(chatQueryDO.getAgentId()) - .dataSetId(parseInfo.getDataSetId()) - .queryText(queryReq.getQueryText()).build()); + String sql = queryResp == null ? null : queryResp.getSql(); + List> resultList = queryResp == null ? new ArrayList<>() + : queryResp.getResultList(); + List columns = queryResp == null ? new ArrayList<>() : queryResp.getColumns(); + queryResult.setQuerySql(sql); + queryResult.setQueryResults(resultList); + queryResult.setQueryColumns(columns); + queryResult.setQueryMode(parseInfo.getQueryMode()); + queryResult.setQueryState(QueryState.SUCCESS); + + return queryResult; } @Override - public SemanticParseInfo queryContext(QueryReq queryCtx) { - ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId()); + public SemanticParseInfo queryContext(Integer chatId) { + ChatContext context = chatContextService.getOrCreateContext(chatId); return context.getParseInfo(); } //mainly used for executing after revising filters,for example:"fans_cnt>=100000"->"fans_cnt>500000", //"style='流行'"->"style in ['流行','爱国']" @Override - @TimeCost - public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException { - ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(), - queryData.getParseId()); - SemanticParseInfo parseInfo = getSemanticParseInfo(queryData, chatParseDO); - SemanticSchema semanticSchema = schemaService.getSemanticSchema(); + public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception { + SemanticParseInfo parseInfo = getSemanticParseInfo(queryData); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); semanticQuery.setParseInfo(parseInfo); + List fields = new ArrayList<>(); if (Objects.nonNull(parseInfo.getSqlInfo()) && StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectS2SQL())) { @@ -334,9 +256,12 @@ public class QueryServiceImpl implements QueryService { String correctorSql = reviseCorrectS2SQL(queryData, parseInfo); parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql); semanticQuery.setParseInfo(parseInfo); - String explainSql = semanticQuery.explain(user); - if (StringUtils.isNotBlank(explainSql)) { - parseInfo.getSqlInfo().setQuerySQL(explainSql); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + ExplainSqlReq explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq) + .queryTypeEnum(QueryMethod.SQL).build(); + ExplainResp explain = queryService.explain(explainSqlReq, user); + if (StringUtils.isNotBlank(explain.getSql())) { + parseInfo.getSqlInfo().setQuerySQL(explain.getSql()); } } else { log.info("rule begin replace metrics and revise filters!"); @@ -349,7 +274,8 @@ public class QueryServiceImpl implements QueryService { queryReq.setQueryFilters(new QueryFilters()); queryReq.setUser(user); } - QueryResult queryResult = semanticQuery.execute(user); + SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); + QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user); queryResult.setChatContext(semanticQuery.getParseInfo()); DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); @@ -417,12 +343,10 @@ public class QueryServiceImpl implements QueryService { } @Override - public EntityInfo getEntityInfo(Long queryId, Integer parseId, User user) { - ChatParseDO chatParseDO = chatService.getParseInfo(queryId, parseId); - SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); DataSetSchema dataSetSchema = - schemaService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); return semanticService.getEntityInfo(parseInfo, dataSetSchema, user); } @@ -600,8 +524,8 @@ public class QueryServiceImpl implements QueryService { }); } - private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) { - SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData) { + SemanticParseInfo parseInfo = queryData.getParseInfo(); if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { return parseInfo; } @@ -643,7 +567,7 @@ public class QueryServiceImpl implements QueryService { SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID()); Set detectDataSetIds = new HashSet<>(); detectDataSetIds.add(schemaElement.getDataSet()); - dimensionValueReq.setModelId(schemaElement.getDataSet()); + dimensionValueReq.setModelId(schemaElement.getModel()); List dimensionValues = getDimensionValues(dimensionValueReq, detectDataSetIds); // if the search results is null,search dimensionValue from database if (CollectionUtils.isEmpty(dimensionValues)) { @@ -673,9 +597,11 @@ public class QueryServiceImpl implements QueryService { if (StringUtils.isBlank(dimensionValueReq.getValue())) { return SearchService.getDimensionValue(dimensionValueReq); } + Map> modelIdToDataSetIds = new HashMap<>(); + modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds)); //search from prefixSearch List hanlpMapResultList = knowledgeService.prefixSearch(dimensionValueReq.getValue(), - 2000, dataSetIds); + 2000, modelIdToDataSetIds); HanlpHelper.transLetterOriginal(hanlpMapResultList); return hanlpMapResultList.stream() .filter(o -> { @@ -691,7 +617,7 @@ public class QueryServiceImpl implements QueryService { .collect(Collectors.toList()); } - private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) { + private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) throws Exception { QueryStructReq queryStructReq = new QueryStructReq(); DateConf dateConf = new DateConf(); @@ -705,8 +631,7 @@ public class QueryServiceImpl implements QueryService { List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); queryStructReq.setGroups(groups); - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - return semanticInterpreter.queryByStruct(queryStructReq, user); + return queryService.queryByReq(queryStructReq, user); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java index 76d829616..0c03a0e19 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java @@ -239,6 +239,11 @@ public class DataSetServiceImpl Collectors.mapping(Pair::getRight, Collectors.toList()))); } + @Override + public Map> getModelIdToDataSetIds() { + return getModelIdToDataSetIds(Lists.newArrayList()); + } + private void conflictCheck(DataSetResp dataSetResp) { List allDimensionIds = dataSetResp.getAllDimensions(); List allMetricIds = dataSetResp.getAllMetrics(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 78f112834..71f9f8b53 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -9,11 +9,11 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.core.file.FileHandler; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; import com.tencent.supersonic.headless.server.service.DictTaskService; -import com.tencent.supersonic.headless.server.service.KnowledgeService; import com.tencent.supersonic.headless.server.utils.DictUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -41,7 +41,6 @@ public class DictTaskServiceImpl implements DictTaskService { private final DictUtils dictConverter; private final DictUtils dictUtils; private final FileHandler fileHandler; - private final KnowledgeService knowledgeService; public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter, @@ -52,7 +51,6 @@ public class DictTaskServiceImpl implements DictTaskService { this.dictConverter = dictConverter; this.dictUtils = dictUtils; this.fileHandler = fileHandler; - this.knowledgeService = knowledgeService; } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index f7b796eb9..284b2c0c3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -458,14 +458,14 @@ public class MetricServiceImpl implements MetricService { if (bizNameMap.containsKey(metricReq.getBizName())) { MetricResp metricResp = bizNameMap.get(metricReq.getBizName()); if (!metricResp.getId().equals(metricReq.getId())) { - throw new RuntimeException(String.format("该主题域下存在相同的指标字段名:%s 创建人:%s", + throw new RuntimeException(String.format("该模型下存在相同的指标字段名:%s 创建人:%s", metricReq.getBizName(), metricResp.getCreatedBy())); } } if (nameMap.containsKey(metricReq.getName())) { MetricResp metricResp = nameMap.get(metricReq.getName()); if (!metricResp.getId().equals(metricReq.getId())) { - throw new RuntimeException(String.format("该主题域下存在相同的指标名:%s 创建人:%s", + throw new RuntimeException(String.format("该模型下存在相同的指标名:%s 创建人:%s", metricReq.getName(), metricResp.getCreatedBy())); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index 66c0bfed9..549be85c7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.headless.server.service.impl; -import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; - import com.github.pagehelper.PageInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -13,6 +11,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; @@ -44,23 +43,28 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.service.TagMetaService; +import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.utils.TagConverter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; + @Slf4j @Service public class SchemaServiceImpl implements SchemaService { @@ -118,6 +122,43 @@ public class SchemaServiceImpl implements SchemaService { return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst().orElse(null); } + private List fetchDataSetSchema(List ids) { + DataSetFilterReq dataSetFilterReq = new DataSetFilterReq(); + dataSetFilterReq.setDataSetIds(ids); + return fetchDataSetSchema(dataSetFilterReq); + } + + @Override + public DataSetSchema getDataSetSchema(Long dataSetId) { + List ids = new ArrayList<>(); + ids.add(dataSetId); + List dataSetSchemaResps = fetchDataSetSchema(ids); + if (!CollectionUtils.isEmpty(dataSetSchemaResps)) { + Optional dataSetSchemaResp = dataSetSchemaResps.stream() + .filter(d -> d.getId().equals(dataSetId)).findFirst(); + if (dataSetSchemaResp.isPresent()) { + DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get(); + return DataSetSchemaBuilder.build(dataSetSchema); + } + } + return null; + } + + @Override + public List getDataSetSchema() { + return getDataSetSchema(new ArrayList<>()); + } + + public List getDataSetSchema(List ids) { + List domainSchemaList = new ArrayList<>(); + + for (DataSetSchemaResp resp : fetchDataSetSchema(ids)) { + domainSchemaList.add(DataSetSchemaBuilder.build(resp)); + } + + return domainSchemaList; + } + public List buildDataSetSchema(DataSetFilterReq filter) { MetaFilter metaFilter = new MetaFilter(); metaFilter.setStatus(StatusEnum.ONLINE.getCode()); @@ -284,13 +325,6 @@ public class SchemaServiceImpl implements SchemaService { return modelService.getModelListWithAuth(user, domainId, authTypeEnum); } - @Override - public List getDataSetList(Long domainId) { - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setDomainId(domainId); - return dataSetService.getDataSetList(metaFilter); - } - public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) { SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java similarity index 72% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java index 6d2f7eacb..4062c630f 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java @@ -1,41 +1,33 @@ -package com.tencent.supersonic.chat.server.service.impl; +package com.tencent.supersonic.headless.server.service.impl; -import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.SearchResult; -import com.tencent.supersonic.chat.core.agent.Agent; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.response.S2Term; +import com.tencent.supersonic.headless.api.pojo.response.SearchResult; +import com.tencent.supersonic.headless.core.chat.mapper.MatchText; +import com.tencent.supersonic.headless.core.chat.mapper.ModelWithSemanticType; +import com.tencent.supersonic.headless.core.chat.mapper.SearchMatchStrategy; +import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat; import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat; -import com.tencent.supersonic.chat.core.mapper.MapperHelper; -import com.tencent.supersonic.chat.core.mapper.MatchText; -import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType; -import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy; -import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.knowledge.KnowledgeService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; -import com.tencent.supersonic.chat.server.service.AgentService; -import com.tencent.supersonic.chat.server.service.ChatService; -import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.chat.server.service.SearchService; -import com.tencent.supersonic.common.pojo.enums.DictWordType; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.server.service.KnowledgeService; +import com.tencent.supersonic.headless.server.service.ChatContextService; +import com.tencent.supersonic.headless.server.service.DataSetService; +import com.tencent.supersonic.headless.server.service.SearchService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; - import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -59,54 +51,38 @@ public class SearchServiceImpl implements SearchService { private static final int RESULT_SIZE = 10; @Autowired - private SchemaService schemaService; - @Autowired - private ChatService chatService; + private SemanticService semanticService; @Autowired private SearchMatchStrategy searchMatchStrategy; @Autowired - private AgentService agentService; - @Autowired - @Qualifier("searchCaffeineCache") - private Cache caffeineCache; - - @Autowired - private ConfigService configService; - + private ChatContextService chatContextService; @Autowired private KnowledgeService knowledgeService; + @Autowired + private DataSetService dataSetService; @Override public List search(QueryReq queryReq) { - // 1. check search enable - Integer agentId = queryReq.getAgentId(); - if (agentId != null) { - Agent agent = agentService.getAgent(agentId); - if (!agent.enableSearch()) { - return Lists.newArrayList(); - } - } String queryText = queryReq.getQueryText(); - // 2.get meta info - SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema(); + // 1.get meta info + SemanticSchema semanticSchemaDb = semanticService.getSemanticSchema(); List metricsDb = semanticSchemaDb.getMetrics(); - final Map modelToName = semanticSchemaDb.getDataSetIdToName(); - - // 3.detect by segment - List originals = knowledgeService.getTerms(queryText); + final Map dataSetIdToName = semanticSchemaDb.getDataSetIdToName(); + Map> modelIdToDataSetIds = + dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIdToName.keySet())); + // 2.detect by segment + List originals = knowledgeService.getTerms(queryText, modelIdToDataSetIds); log.info("hanlp parse result: {}", originals); - MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); - Set detectDataSetIds = mapperHelper.getDataSetIds(queryReq.getDataSetId(), - agentService.getAgent(agentId)); + Set dataSetIds = queryReq.getDataSetIds(); QueryContext queryContext = new QueryContext(); BeanUtils.copyProperties(queryReq, queryContext); Map> regTextMap = - searchMatchStrategy.match(queryContext, originals, detectDataSetIds); + searchMatchStrategy.match(queryContext, originals, dataSetIds); regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue())); - // 4.get the most matching data + // 3.get the most matching data Optional>> mostSimilarSearchResult = regTextMap.entrySet() .stream() .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) @@ -114,7 +90,7 @@ public class SearchServiceImpl implements SearchService { entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length() ? entry1 : entry2); - // 5.optimize the results after the query + // 4.optimize the results after the query if (!mostSimilarSearchResult.isPresent()) { return Lists.newArrayList(); } @@ -122,12 +98,12 @@ public class SearchServiceImpl implements SearchService { log.info("searchTextEntry:{},queryReq:{}", searchTextEntry, queryReq); Set searchResults = new LinkedHashSet(); - DataSetInfoStat modelStat = NatureHelper.getDataSetStat(originals); + DataSetInfoStat dataSetInfoStat = NatureHelper.getDataSetStat(originals); - List possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getDataSetId()); + List possibleModels = getPossibleDataSets(queryReq, originals, dataSetInfoStat, dataSetIds); // 5.1 priority dimension metric - boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleModels), modelToName, + boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleModels), dataSetIdToName, searchTextEntry, searchResults); // 5.2 process based on dimension values @@ -137,8 +113,8 @@ public class SearchServiceImpl implements SearchService { for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) { - Set searchResultSet = searchDimensionValue(metricsDb, modelToName, - modelStat.getMetricDataSetCount(), existMetricAndDimension, + Set searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName, + dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension, matchText, natureToNameMap, natureToNameEntry, queryReq.getQueryFilters()); searchResults.addAll(searchResultSet); @@ -146,23 +122,21 @@ public class SearchServiceImpl implements SearchService { return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList()); } - private List getPossibleModels(QueryReq queryCtx, List originals, - DataSetInfoStat modelStat, Long webModelId) { - - if (Objects.nonNull(webModelId) && webModelId > 0) { - List result = new ArrayList<>(); - result.add(webModelId); - return result; + private List getPossibleDataSets(QueryReq queryCtx, List originals, + DataSetInfoStat dataSetInfoStat, Set dataSetIds) { + if (CollectionUtils.isNotEmpty(dataSetIds)) { + return new ArrayList<>(dataSetIds); } List possibleModels = NatureHelper.selectPossibleDataSets(originals); - Long contextModel = chatService.getContextModel(queryCtx.getChatId()); + Long contextModel = chatContextService.getContextModel(queryCtx.getChatId()); - log.debug("possibleModels:{},modelStat:{},contextModel:{}", possibleModels, modelStat, contextModel); + log.debug("possibleModels:{},dataSetInfoStat:{},contextModel:{}", + possibleModels, dataSetInfoStat, contextModel); // If nothing is recognized or only metric are present, then add the contextModel. - if (nothingOrOnlyMetric(modelStat)) { + if (nothingOrOnlyMetric(dataSetInfoStat)) { return Lists.newArrayList(contextModel); } return possibleModels; @@ -204,25 +178,16 @@ public class SearchServiceImpl implements SearchService { .schemaElementType(schemaElementType) .subRecommend(wordName) .build(); - ItemNameVisibilityInfo visibility = (ItemNameVisibilityInfo) caffeineCache.getIfPresent(modelId); - if (visibility == null) { - visibility = configService.getVisibilityByModelId(modelId); - caffeineCache.put(modelId, visibility); - } - if (visibility.getBlackMetricNameList().contains(searchResult.getRecommend()) - || visibility.getBlackDimNameList().contains(searchResult.getRecommend())) { - return searchResults; - } + + if (metricModelCount <= 0 && !existMetricAndDimension) { if (filterByQueryFilter(wordName, queryFilters)) { return searchResults; } searchResults.add(searchResult); int metricSize = getMetricSize(natureToNameMap); - //invisibility to filter metrics - List blackMetricNameList = visibility.getBlackMetricNameList(); List metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3) - .stream().filter(o -> !blackMetricNameList.contains(o)) + .stream() .limit(metricSize).collect(Collectors.toList()); for (String metric : metrics) { @@ -333,15 +298,7 @@ public class SearchServiceImpl implements SearchService { .schemaElementType(schemaElementType) .build(); //visibility to filter metrics - ItemNameVisibilityInfo visibility = (ItemNameVisibilityInfo) caffeineCache.getIfPresent(modelId); - if (visibility == null) { - visibility = configService.getVisibilityByModelId(modelId); - caffeineCache.put(modelId, visibility); - } - if (!visibility.getBlackMetricNameList().contains(hanlpMapResult.getName()) - && !visibility.getBlackDimNameList().contains(hanlpMapResult.getName())) { - searchResults.add(searchResult); - } + searchResults.add(searchResult); } log.info("parseResult:{},dimensionMetricClassIds:{},possibleModels:{}", hanlpMapResult, dimensionMetricClassIds, possibleModels); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java similarity index 90% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java index f6d79a51d..cad524df5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java @@ -1,27 +1,26 @@ -package com.tencent.supersonic.chat.server.service; +package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.DataInfo; -import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.DataSetInfo; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.DataInfo; +import com.tencent.supersonic.headless.api.pojo.DataSetInfo; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -44,10 +43,11 @@ public class SemanticService { @Autowired private SchemaService schemaService; - private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); + @Autowired + private QueryService queryService; public SemanticSchema getSemanticSchema() { - return schemaService.getSemanticSchema(); + return new SemanticSchema(schemaService.getDataSetSchema()); } public DataSetSchema getDataSetSchema(Long id) { @@ -169,7 +169,7 @@ public class SemanticService { } semanticParseInfo.setDateInfo(dateInfo); - // add filter + //add filter QueryFilter chatFilter = getQueryFilter(entityInfo); Set chatFilters = new LinkedHashSet(); chatFilters.add(chatFilter); @@ -178,7 +178,7 @@ public class SemanticService { SemanticQueryResp queryResultWithColumns = null; try { QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo); - queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructReq, user); + queryResultWithColumns = queryService.queryByReq(queryStructReq, user); } catch (Exception e) { log.warn("setMainModel queryByStruct error, e:", e); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/WordService.java similarity index 81% rename from chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/WordService.java index fadcdcd2e..0889694b5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/WordService.java @@ -1,13 +1,13 @@ -package com.tencent.supersonic.chat.server.service.impl; +package com.tencent.supersonic.headless.server.service.impl; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; +import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -20,11 +20,13 @@ import java.util.stream.Collectors; @Slf4j public class WordService { + @Autowired + private SchemaService schemaService; + private List preDictWords = new ArrayList<>(); public List getAllDictWords() { - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getDataSetSchema()); + SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema()); List words = new ArrayList<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java new file mode 100644 index 000000000..8219cb1da --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ComponentFactory.java @@ -0,0 +1,58 @@ +package com.tencent.supersonic.headless.server.utils; + +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector; +import com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper; +import com.tencent.supersonic.headless.core.chat.parser.SemanticParser; +import com.tencent.supersonic.headless.server.processor.ResultProcessor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.core.io.support.SpringFactoriesLoader; + +import java.util.ArrayList; +import java.util.List; + +/** + * HeadlessConverter QueryOptimizer QueryExecutor object factory + */ +@Slf4j +public class ComponentFactory { + private static List resultProcessors = new ArrayList<>(); + private static List schemaMappers = new ArrayList<>(); + private static List semanticParsers = new ArrayList<>(); + private static List semanticCorrectors = new ArrayList<>(); + + public static List getResultProcessors() { + return CollectionUtils.isEmpty(resultProcessors) ? init(ResultProcessor.class, + resultProcessors) : resultProcessors; + } + + public static List getSchemaMappers() { + return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers; + } + + public static List getSemanticParsers() { + return CollectionUtils.isEmpty(semanticParsers) ? init(SemanticParser.class, semanticParsers) : semanticParsers; + } + + public static List getSemanticCorrectors() { + return CollectionUtils.isEmpty(semanticCorrectors) ? init(SemanticCorrector.class, + semanticCorrectors) : semanticCorrectors; + } + + public static T getBean(String name, Class tClass) { + return ContextUtils.getContext().getBean(name, tClass); + } + + private static List init(Class factoryType, List list) { + list.addAll(SpringFactoriesLoader.loadFactories(factoryType, + Thread.currentThread().getContextClassLoader())); + return list; + } + + private static T init(Class factoryType) { + return SpringFactoriesLoader.loadFactories(factoryType, + Thread.currentThread().getContextClassLoader()).get(0); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java similarity index 98% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 408df9f9a..fade71ae4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -1,19 +1,18 @@ -package com.tencent.supersonic.chat.core.query.semantic; +package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; -import com.tencent.supersonic.chat.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.DimValueMap; -import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; +import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; -import java.util.Objects; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; @@ -22,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 4d1f6a0a1..4e4ba2485 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -1,11 +1,5 @@ package com.tencent.supersonic.headless.server.utils; -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.POUND; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; - import com.google.common.base.Strings; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; @@ -40,7 +34,13 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; - +import com.tencent.supersonic.headless.server.service.TagMetaService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -53,14 +53,11 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; - -import com.tencent.supersonic.headless.server.service.TagMetaService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.POUND; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; @Slf4j @Component @@ -298,6 +295,7 @@ public class DictUtils { } private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { + // todo tag String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; String bizName = dictItemResp.getBizName(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index b77e7de39..293f4507a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.QueryOptMode; -import com.tencent.supersonic.headless.api.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; @@ -110,7 +110,7 @@ public class StatUtils { queryStatInfo.setTraceId(traceId) .setDataSetId(queryTagReq.getDataSetId()) .setUser(user) - .setQueryType(QueryType.STRUCT.getValue()) + .setQueryType(QueryMethod.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQueryStructCmd(queryTagReq.toString()) .setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString())) @@ -147,7 +147,7 @@ public class StatUtils { queryStatInfo.setTraceId("") .setUser(userName) .setDataSetId(querySqlReq.getDataSetId()) - .setQueryType(QueryType.SQL.getValue()) + .setQueryType(QueryMethod.SQL.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQuerySqlCmd(querySqlReq.toString()) .setQuerySqlCmdMd5(DigestUtils.md5Hex(querySqlReq.toString())) @@ -178,7 +178,7 @@ public class StatUtils { queryStatInfo.setTraceId(traceId) .setDataSetId(queryStructReq.getDataSetId()) .setUser(user) - .setQueryType(QueryType.STRUCT.getValue()) + .setQueryType(QueryMethod.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQueryStructCmd(queryStructReq.toString()) .setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructReq.toString())) diff --git a/chat/server/src/main/resources/mapper/ChatContextMapper.xml b/headless/server/src/main/resources/mapper/ChatContextMapper.xml similarity index 81% rename from chat/server/src/main/resources/mapper/ChatContextMapper.xml rename to headless/server/src/main/resources/mapper/ChatContextMapper.xml index b957115f8..c00ef253e 100644 --- a/chat/server/src/main/resources/mapper/ChatContextMapper.xml +++ b/headless/server/src/main/resources/mapper/ChatContextMapper.xml @@ -3,10 +3,10 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> - + + type="com.tencent.supersonic.headless.server.persistence.dataobject.ChatContextDO"> @@ -20,7 +20,7 @@ from s2_chat_context where chat_id=#{chatId} limit 1 - + insert into s2_chat_context (chat_id,user,query_text,semantic_parse) values (#{chatId}, #{user},#{queryText}, #{semanticParse}) diff --git a/chat/server/src/main/resources/mapper/StatisticsMapper.xml b/headless/server/src/main/resources/mapper/StatisticsMapper.xml similarity index 70% rename from chat/server/src/main/resources/mapper/StatisticsMapper.xml rename to headless/server/src/main/resources/mapper/StatisticsMapper.xml index a6342ea57..167138dc1 100644 --- a/chat/server/src/main/resources/mapper/StatisticsMapper.xml +++ b/headless/server/src/main/resources/mapper/StatisticsMapper.xml @@ -3,9 +3,9 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> - + - + @@ -16,7 +16,7 @@ - + insert into s2_chat_statistics (question_id,chat_id, user_name, query_text, interface_name,cost,type ,create_time) values @@ -25,11 +25,4 @@ - - diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryReqBuilderTest.java similarity index 97% rename from chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java rename to headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryReqBuilderTest.java index 6e3e7c66a..957b58fcb 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryReqBuilderTest.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.server.utils; +package com.tencent.supersonic.headless.server.utils; import com.tencent.supersonic.common.pojo.Aggregator; @@ -12,10 +12,10 @@ import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import org.junit.Assert; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.testng.Assert; import java.util.ArrayList; import java.util.Arrays; diff --git a/launchers/chat/pom.xml b/launchers/chat/pom.xml index 3cc1d1875..841cb4e42 100644 --- a/launchers/chat/pom.xml +++ b/launchers/chat/pom.xml @@ -24,12 +24,6 @@ ${project.version} - - com.tencent.supersonic - chat-core - ${project.version} - - com.tencent.supersonic auth-authorization diff --git a/launchers/standalone/pom.xml b/launchers/standalone/pom.xml index 804410b95..65e19dfc4 100644 --- a/launchers/standalone/pom.xml +++ b/launchers/standalone/pom.xml @@ -22,12 +22,6 @@ launchers-common ${project.version} - - - com.tencent.supersonic - chat-core - ${project.version} - com.tencent.supersonic chat-server diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index 537311f30..27df553a6 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -3,30 +3,28 @@ package com.tencent.supersonic; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentConfig; -import com.tencent.supersonic.chat.core.agent.AgentToolType; -import com.tencent.supersonic.chat.core.agent.LLMParserTool; -import com.tencent.supersonic.chat.core.agent.RuleParserTool; -import com.tencent.supersonic.chat.core.plugin.Plugin; -import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; -import com.tencent.supersonic.chat.core.query.plugin.ParamOption; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; +import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.agent.Agent; +import com.tencent.supersonic.chat.server.agent.AgentConfig; +import com.tencent.supersonic.chat.server.agent.AgentToolType; +import com.tencent.supersonic.chat.server.agent.LLMParserTool; +import com.tencent.supersonic.chat.server.agent.RuleParserTool; +import com.tencent.supersonic.chat.server.plugin.Plugin; +import com.tencent.supersonic.chat.server.plugin.PluginParseConfig; +import com.tencent.supersonic.chat.server.plugin.build.ParamOption; +import com.tencent.supersonic.chat.server.plugin.build.WebBase; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.service.PluginService; -import com.tencent.supersonic.chat.server.service.QueryService; import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import java.util.Arrays; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; @@ -39,9 +37,6 @@ import org.springframework.util.CollectionUtils; public class ChatDemoLoader implements CommandLineRunner { private User user = User.getFakeUser(); - @Qualifier("chatQueryService") - @Autowired - private QueryService queryService; @Autowired private ChatService chatService; @Autowired @@ -84,26 +79,24 @@ public class ChatDemoLoader implements CommandLineRunner { } private void parseAndExecute(int chatId, String queryText) throws Exception { - QueryReq queryRequest = new QueryReq(); - queryRequest.setQueryText(queryText); - queryRequest.setChatId(chatId); - queryRequest.setAgentId(1); - queryRequest.setUser(User.getFakeUser()); - ParseResp parseResp = queryService.performParsing(queryRequest); + ChatParseReq chatParseReq = new ChatParseReq(); + chatParseReq.setQueryText(queryText); + chatParseReq.setChatId(chatId); + chatParseReq.setAgentId(1); + chatParseReq.setUser(User.getFakeUser()); + ParseResp parseResp = chatService.performParsing(chatParseReq); if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) { log.info("parseResp.getSelectedParses() is empty"); return; } - ExecuteQueryReq executeReq = ExecuteQueryReq.builder().build(); + ChatExecuteReq executeReq = new ChatExecuteReq(); executeReq.setQueryId(parseResp.getQueryId()); executeReq.setParseId(parseResp.getSelectedParses().get(0).getId()); - executeReq.setQueryText(queryRequest.getQueryText()); - executeReq.setParseInfo(parseResp.getSelectedParses().get(0)); + executeReq.setQueryText(queryText); executeReq.setChatId(parseResp.getChatId()); - executeReq.setUser(queryRequest.getUser()); - executeReq.setAgentId(1); + executeReq.setUser(User.getFakeUser()); executeReq.setSaveAnswer(true); - queryService.performExecution(executeReq); + chatService.performExecution(executeReq); } public void addSampleChats() throws Exception { diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java b/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java index efdd52353..2b3b5715f 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/EmbeddingInitListener.java @@ -1,16 +1,16 @@ package com.tencent.supersonic; -import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; -import com.tencent.supersonic.chat.core.parser.sql.llm.SqlExamplarLoader; -import com.tencent.supersonic.chat.core.parser.sql.llm.SqlExample; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.config.EmbeddingConfig; -import java.util.List; +import com.tencent.supersonic.headless.core.chat.parser.JavaLLMProxy; +import com.tencent.supersonic.headless.core.chat.parser.llm.SqlExamplarLoader; +import com.tencent.supersonic.headless.core.chat.parser.llm.SqlExample; +import com.tencent.supersonic.headless.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import java.util.List; @Slf4j @Component diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 9148f838f..3f781587a 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -1,35 +1,28 @@ -com.tencent.supersonic.chat.core.mapper.SchemaMapper=\ - com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \ - com.tencent.supersonic.chat.core.mapper.KeywordMapper, \ - com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.core.mapper.EntityMapper +com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper=\ + com.tencent.supersonic.headless.core.chat.mapper.EmbeddingMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.KeywordMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.QueryFilterMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.EntityMapper -com.tencent.supersonic.chat.core.parser.SemanticParser=\ - com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \ - com.tencent.supersonic.chat.core.parser.sql.llm.LLMSqlParser, \ - com.tencent.supersonic.chat.core.parser.plugin.embedding.EmbeddingRecallParser, \ - com.tencent.supersonic.chat.core.parser.plugin.function.FunctionCallParser, \ - com.tencent.supersonic.chat.core.parser.QueryTypeParser +com.tencent.supersonic.headless.core.chat.parser.SemanticParser=\ + com.tencent.supersonic.headless.core.chat.parser.rule.RuleSqlParser, \ + com.tencent.supersonic.headless.core.chat.parser.llm.LLMSqlParser, \ + com.tencent.supersonic.headless.core.chat.parser.QueryTypeParser -com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ - com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \ - com.tencent.supersonic.chat.core.corrector.TimeCorrector, \ - com.tencent.supersonic.chat.core.corrector.GrammarCorrector +com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector=\ + com.tencent.supersonic.headless.core.chat.corrector.SchemaCorrector, \ + com.tencent.supersonic.headless.core.chat.corrector.TimeCorrector, \ + com.tencent.supersonic.headless.core.chat.corrector.GrammarCorrector -com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ - com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.QueryRankProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.SqlInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor +com.tencent.supersonic.headless.server.processor.ResultProcessor=\ + com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \ + com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \ + com.tencent.supersonic.headless.server.processor.SqlInfoProcessor, \ + com.tencent.supersonic.headless.server.processor.TimeCostProcessor, \ + com.tencent.supersonic.headless.server.processor.RespBuildProcessor -com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ - com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter - -com.tencent.supersonic.chat.core.parser.sql.llm.DataSetResolver=\ - com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicDataSetResolver +com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\ + com.tencent.supersonic.headless.core.chat.parser.llm.HeuristicDataSetResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java index b09408519..e9209e987 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java @@ -1,21 +1,18 @@ package com.tencent.supersonic.chat; import com.tencent.supersonic.BaseApplication; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.util.DataUtils; +import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.chat.server.service.QueryService; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.api.pojo.response.QueryState; +import com.tencent.supersonic.util.DataUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; import java.time.LocalDate; @@ -32,9 +29,6 @@ public class BaseTest extends BaseApplication { protected final String endDay = LocalDate.now().plusDays(-1).toString(); protected final String period = "DAY"; - @Autowired - @Qualifier("chatQueryService") - protected QueryService queryService; @Autowired protected ChatService chatService; @Autowired @@ -46,17 +40,15 @@ public class BaseTest extends BaseApplication { ParseResp parseResp = submitParse(queryText, agentId, chatId); SemanticParseInfo semanticParseInfo = parseResp.getSelectedParses().get(0); - ExecuteQueryReq request = ExecuteQueryReq.builder() - .agentId(agentId) - .queryId(parseResp.getQueryId()) - .parseId(semanticParseInfo.getId()) + ChatExecuteReq request = ChatExecuteReq.builder() .chatId(parseResp.getChatId()) .queryText(parseResp.getQueryText()) .user(DataUtils.getUser()) - .parseInfo(semanticParseInfo) + .parseId(semanticParseInfo.getId()) + .queryId(parseResp.getQueryId()) .saveAnswer(true) .build(); - QueryResult queryResult = queryService.performExecution(request); + QueryResult queryResult = chatService.performExecution(request); queryResult.setChatContext(semanticParseInfo); return queryResult; } @@ -65,22 +57,16 @@ public class BaseTest extends BaseApplication { ParseResp parseResp = submitParse(queryText, agentId); SemanticParseInfo parseInfo = parseResp.getSelectedParses().get(0); - ExecuteQueryReq request = ExecuteQueryReq.builder() - .agentId(agentId) - .queryId(parseResp.getQueryId()) - .parseId(parseInfo.getId()) + ChatExecuteReq request = ChatExecuteReq.builder() .chatId(parseResp.getChatId()) .queryText(parseResp.getQueryText()) .user(DataUtils.getUser()) - .parseInfo(parseInfo) - .saveAnswer(true) + .parseId(parseInfo.getId()) + .queryId(parseResp.getQueryId()) + .saveAnswer(false) .build(); - QueryResult result = queryService.performExecution(request); - - ChatContext chatContext = chatService.getOrCreateContext(parseResp.getChatId()); - chatContext.setParseInfo(new SemanticParseInfo()); - chatService.updateContext(chatContext); + QueryResult result = chatService.performExecution(request); result.setChatContext(parseInfo); return result; } @@ -89,9 +75,9 @@ public class BaseTest extends BaseApplication { if (Objects.isNull(chatId)) { chatId = 10; } - QueryReq queryContextReq = DataUtils.getQueryContextReq(chatId, queryText); - queryContextReq.setAgentId(agentId); - return queryService.performParsing(queryContextReq); + ChatParseReq chatParseReq = DataUtils.getChatParseReq(chatId, queryText); + chatParseReq.setAgentId(agentId); + return chatService.performParsing(chatParseReq); } protected ParseResp submitParse(String queryText, Integer agentId) { @@ -99,8 +85,8 @@ public class BaseTest extends BaseApplication { } protected ParseResp submitParseWithAgent(String queryText, Integer agentId) { - QueryReq queryContextReq = DataUtils.getQueryReqWithAgent(10, queryText, agentId); - return queryService.performParsing(queryContextReq); + ChatParseReq chatParseReq = DataUtils.getChatParseReqWithAgent(10, queryText, agentId); + return chatService.performParsing(chatParseReq); } protected void assertSchemaElements(Set expected, Set actual) { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java index cc727d2fb..93d7b1923 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java @@ -1,27 +1,28 @@ package com.tencent.supersonic.chat; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; - -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricFilterQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricGroupByQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricModelQuery; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricTopNQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.response.ParseResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricFilterQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricGroupByQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricModelQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricTopNQuery; import com.tencent.supersonic.util.DataUtils; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.Test; + +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; public class MetricTest extends BaseTest { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java index b1a3421a2..bbf2d45ce 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.plugin.PluginManager; +import com.tencent.supersonic.chat.server.plugin.PluginManager; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.embedding.Retrieval; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java index 5e9e5c372..648cfba75 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java @@ -1,17 +1,17 @@ package com.tencent.supersonic.chat; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; - -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricFilterQuery; -import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricFilterQuery; +import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; + public class MultiTurnsTest extends BaseTest { @Test diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java index 23cf3d0ed..486bfd8fc 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java @@ -1,17 +1,17 @@ package com.tencent.supersonic.chat; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery; -import com.tencent.supersonic.chat.core.query.rule.tag.TagFilterQuery; -import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricTagQuery; +import com.tencent.supersonic.headless.core.chat.query.rule.tag.TagFilterQuery; +import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java index 5e21a0180..dd4bf2b3b 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java @@ -1,16 +1,16 @@ package com.tencent.supersonic.chat.mapper; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery; import com.tencent.supersonic.chat.BaseTest; -import com.tencent.supersonic.util.DataUtils; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; +import com.tencent.supersonic.headless.api.pojo.response.QueryResult; +import com.tencent.supersonic.headless.core.chat.query.rule.metric.MetricTagQuery; +import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Test; import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; @@ -20,8 +20,8 @@ public class MapperTest extends BaseTest { @Test public void hanlp() throws Exception { - QueryReq queryContextReq = DataUtils.getQueryContextReq(10, "艺人周杰伦的播放量"); - queryContextReq.setAgentId(DataUtils.tagAgentId); + ChatParseReq chatParseReq = DataUtils.getChatParseReq(10, "艺人周杰伦的播放量"); + chatParseReq.setAgentId(DataUtils.tagAgentId); QueryResult actualResult = submitNewChat("艺人周杰伦的播放量", DataUtils.tagAgentId); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/BasePluginTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/BasePluginTest.java deleted file mode 100644 index dcaef45f8..000000000 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/BasePluginTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.chat.plugin; - -import com.tencent.supersonic.BaseApplication; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.core.query.plugin.WebBase; -import com.tencent.supersonic.chat.core.query.plugin.webpage.WebPageQuery; -import com.tencent.supersonic.chat.core.query.plugin.webpage.WebPageResp; -import org.junit.Assert; - -public class BasePluginTest extends BaseApplication { - - protected void assertPluginRecognizeResult(QueryResult queryResult) { - Assert.assertEquals(queryResult.getQueryState(), QueryState.SUCCESS); - Assert.assertEquals(queryResult.getQueryMode(), WebPageQuery.QUERY_MODE); - WebPageResp webPageResponse = (WebPageResp) queryResult.getResponse(); - WebBase webPage = webPageResponse.getWebPage(); - Assert.assertEquals(webPage.getUrl(), "www.yourbi.com"); - Assert.assertEquals(1, webPage.getParams().size()); - Assert.assertEquals("alice", webPage.getParams().get(0).getValue()); - } - -} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/PluginRecognizeTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/PluginRecognizeTest.java deleted file mode 100644 index 055d40e42..000000000 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/PluginRecognizeTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.tencent.supersonic.chat.plugin; - -import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.plugin.PluginManager; -import com.tencent.supersonic.chat.MockConfiguration; -import com.tencent.supersonic.util.DataUtils; -import com.tencent.supersonic.chat.server.service.AgentService; -import com.tencent.supersonic.chat.server.service.QueryService; -import com.tencent.supersonic.common.config.EmbeddingConfig; -import org.junit.Assert; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.mock.mockito.MockBean; - -public class PluginRecognizeTest extends BasePluginTest { - - @MockBean - protected PluginManager pluginManager; - - @MockBean - private EmbeddingConfig embeddingConfig; - - @MockBean - private AgentService agentService; - - @Autowired - @Qualifier("chatQueryService") - private QueryService queryService; - - public void webPageRecognize() throws Exception { - MockConfiguration.mockEmbeddingRecognize(pluginManager, "alice最近的访问情况怎么样", "1"); - MockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryReq queryContextReq = DataUtils.getQueryReqWithAgent(1000, "alice最近的访问情况怎么样", 1); - - ParseResp parseResp = queryService.performParsing(queryContextReq); - ExecuteQueryReq executeReq = ExecuteQueryReq.builder().user(queryContextReq.getUser()) - .chatId(parseResp.getChatId()) - .queryId(parseResp.getQueryId()) - .queryText(parseResp.getQueryText()) - .parseInfo(parseResp.getSelectedParses().get(0)) - .build(); - QueryResult queryResult = queryService.performExecution(executeReq); - - assertPluginRecognizeResult(queryResult); - } - - public void webPageRecognizeWithQueryFilter() throws Exception { - MockConfiguration.mockEmbeddingRecognize(pluginManager, "在超音数最近的情况怎么样", "1"); - MockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryReq queryRequest = DataUtils.getQueryReqWithAgent(1000, "在超音数最近的情况怎么样", 1); - QueryFilters queryFilters = new QueryFilters(); - QueryFilter queryFilter = new QueryFilter(); - queryFilter.setElementID(2L); - queryFilter.setValue("alice"); - // queryRequest.setModelId(1L); - queryFilters.getFilters().add(queryFilter); - queryRequest.setQueryFilters(queryFilters); - - ParseResp parseResp = queryService.performParsing(queryRequest); - ExecuteQueryReq executeReq = ExecuteQueryReq.builder().user(queryRequest.getUser()) - .chatId(parseResp.getChatId()) - .queryId(parseResp.getQueryId()) - .queryText(parseResp.getQueryText()) - .parseInfo(parseResp.getSelectedParses().get(0)) - .build(); - QueryResult queryResult = queryService.performExecution(executeReq); - - assertPluginRecognizeResult(queryResult); - } - - public void pluginRecognizeWithAgent() { - MockConfiguration.mockEmbeddingRecognize(pluginManager, "alice最近的访问情况怎么样", "1"); - MockConfiguration.mockEmbeddingUrl(embeddingConfig); - MockConfiguration.mockMetricAgent(agentService); - QueryReq queryContextReq = DataUtils.getQueryReqWithAgent(1000, "alice最近的访问情况怎么样", - DataUtils.getMetricAgent().getId()); - ParseResp parseResp = queryService.performParsing(queryContextReq); - Assert.assertTrue(parseResp.getSelectedParses() != null - && parseResp.getSelectedParses().size() > 0); - } - -} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java index 9f232407d..a39b97a06 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java @@ -2,12 +2,12 @@ package com.tencent.supersonic.headless; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.headless.api.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; +import com.tencent.supersonic.headless.core.utils.QueryReqBuilder; import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Test; @@ -22,7 +22,7 @@ public class ExplainTest extends BaseTest { public void testSqlExplain() throws Exception { String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "; ExplainSqlReq explainSqlReq = ExplainSqlReq.builder() - .queryTypeEnum(QueryType.SQL) + .queryTypeEnum(QueryMethod.SQL) .queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView())) .build(); ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser()); @@ -36,7 +36,7 @@ public class ExplainTest extends BaseTest { public void testStructExplain() throws Exception { QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department")); ExplainSqlReq explainSqlReq = ExplainSqlReq.builder() - .queryTypeEnum(QueryType.STRUCT) + .queryTypeEnum(QueryMethod.STRUCT) .queryReq(queryStructReq) .build(); ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser()); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 4e269b69f..8a8756042 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -3,16 +3,17 @@ package com.tencent.supersonic.util; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.core.agent.Agent; -import com.tencent.supersonic.chat.core.agent.AgentConfig; -import com.tencent.supersonic.chat.core.agent.AgentToolType; -import com.tencent.supersonic.chat.core.agent.PluginTool; -import com.tencent.supersonic.chat.core.agent.RuleParserTool; +import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq; +import com.tencent.supersonic.chat.server.agent.Agent; +import com.tencent.supersonic.chat.server.agent.AgentConfig; +import com.tencent.supersonic.chat.server.agent.AgentToolType; +import com.tencent.supersonic.chat.server.agent.PluginTool; +import com.tencent.supersonic.chat.server.agent.RuleParserTool; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; + import java.util.HashSet; import java.util.Set; @@ -41,16 +42,16 @@ public class DataUtils { return User.get(3L, "tom"); } - public static QueryReq getQueryContextReq(Integer id, String query) { - QueryReq queryContextReq = new QueryReq(); - queryContextReq.setQueryText(query); - queryContextReq.setChatId(id); - queryContextReq.setUser(user_test); - return queryContextReq; + public static ChatParseReq getChatParseReq(Integer id, String query) { + ChatParseReq chatParseReq = new ChatParseReq(); + chatParseReq.setQueryText(query); + chatParseReq.setChatId(id); + chatParseReq.setUser(user_test); + return chatParseReq; } - public static QueryReq getQueryReqWithAgent(Integer id, String query, Integer agentId) { - QueryReq queryReq = new QueryReq(); + public static ChatParseReq getChatParseReqWithAgent(Integer id, String query, Integer agentId) { + ChatParseReq queryReq = new ChatParseReq(); queryReq.setQueryText(query); queryReq.setChatId(id); queryReq.setUser(user_test); diff --git a/launchers/standalone/src/test/resources/META-INF/spring.factories b/launchers/standalone/src/test/resources/META-INF/spring.factories index a239827e2..3f781587a 100644 --- a/launchers/standalone/src/test/resources/META-INF/spring.factories +++ b/launchers/standalone/src/test/resources/META-INF/spring.factories @@ -1,35 +1,28 @@ -com.tencent.supersonic.chat.core.mapper.SchemaMapper=\ - com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \ - com.tencent.supersonic.chat.core.mapper.KeywordMapper, \ - com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.core.mapper.EntityMapper +com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper=\ + com.tencent.supersonic.headless.core.chat.mapper.EmbeddingMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.KeywordMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.QueryFilterMapper, \ + com.tencent.supersonic.headless.core.chat.mapper.EntityMapper -com.tencent.supersonic.chat.core.parser.SemanticParser=\ - com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \ - com.tencent.supersonic.chat.core.parser.sql.llm.LLMSqlParser, \ - com.tencent.supersonic.chat.core.parser.plugin.embedding.EmbeddingRecallParser, \ - com.tencent.supersonic.chat.core.parser.plugin.function.FunctionCallParser, \ - com.tencent.supersonic.chat.core.parser.QueryTypeParser +com.tencent.supersonic.headless.core.chat.parser.SemanticParser=\ + com.tencent.supersonic.headless.core.chat.parser.rule.RuleSqlParser, \ + com.tencent.supersonic.headless.core.chat.parser.llm.LLMSqlParser, \ + com.tencent.supersonic.headless.core.chat.parser.QueryTypeParser -com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ - com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \ - com.tencent.supersonic.chat.core.corrector.TimeCorrector, \ - com.tencent.supersonic.chat.core.corrector.GrammarCorrector +com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector=\ + com.tencent.supersonic.headless.core.chat.corrector.SchemaCorrector, \ + com.tencent.supersonic.headless.core.chat.corrector.TimeCorrector, \ + com.tencent.supersonic.headless.core.chat.corrector.GrammarCorrector -com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ - com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.QueryRankProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.SqlInfoProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \ - com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor +com.tencent.supersonic.headless.server.processor.ResultProcessor=\ + com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \ + com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \ + com.tencent.supersonic.headless.server.processor.SqlInfoProcessor, \ + com.tencent.supersonic.headless.server.processor.TimeCostProcessor, \ + com.tencent.supersonic.headless.server.processor.RespBuildProcessor -com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\ - com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter - -com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ - com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver +com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\ + com.tencent.supersonic.headless.core.chat.parser.llm.HeuristicDataSetResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor @@ -39,7 +32,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\ com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\ - com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor + com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\ + com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ - com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file + com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore