From 8d81f63e081408004a7a4a2bd6a5c2947fc9136c Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 18 Oct 2023 09:53:01 +0800 Subject: [PATCH] (feature)(project) change dsl to s2ql in project and queryMode to llmParser (#250) --- .../api/component/SemanticInterpreter.java | 4 +- .../chat/api/pojo/response/SqlInfo.java | 2 +- .../chat/agent/tool/AgentToolType.java | 2 +- .../chat/agent/tool/CommonAgentTool.java | 16 +++++ .../tool/{DslTool.java => LLMParserTool.java} | 4 +- .../chat/agent/tool/RuleQueryTool.java | 3 +- .../chat/corrector/GlobalBeforeCorrector.java | 12 ++-- .../chat/corrector/WhereCorrector.java | 4 +- .../supersonic/chat/mapper/MapperHelper.java | 2 +- .../chat/parser/SatisfactionChecker.java | 4 +- .../{dsl => s2ql}/HeuristicModelResolver.java | 2 +- .../LLMS2QLParser.java} | 67 ++++++++++--------- .../llm/{dsl => s2ql}/ModelMatchResult.java | 2 +- .../llm/{dsl => s2ql}/ModelResolver.java | 2 +- .../ParseResult.java} | 12 ++-- .../S2QLDateHelper.java} | 4 +- .../plugin/function/FunctionBasedParser.java | 4 +- .../chat/query/llm/{dsl => s2ql}/LLMReq.java | 2 +- .../chat/query/llm/{dsl => s2ql}/LLMResp.java | 2 +- .../DslQuery.java => s2ql/S2QLQuery.java} | 29 ++++---- .../execute/EntityInfoExecuteResponder.java | 4 +- .../parse/EntityInfoParseResponder.java | 4 +- .../supersonic/chat/service/AgentService.java | 6 +- .../chat/service/impl/AgentServiceImpl.java | 15 +++-- .../chat/service/impl/QueryServiceImpl.java | 34 +++++----- .../chat/utils/ComponentFactory.java | 7 +- .../chat/utils/QueryReqBuilder.java | 14 ++-- .../LLMS2QLParserTest.java} | 8 +-- .../semantic/LocalSemanticInterpreter.java | 6 +- .../semantic/RemoteSemanticInterpreter.java | 6 +- .../main/resources/META-INF/spring.factories | 6 +- .../com/tencent/supersonic/ConfigureDemo.java | 22 +++--- .../main/resources/META-INF/spring.factories | 6 +- .../src/main/resources/db/sql-update.sql | 2 + pom.xml | 2 +- .../{QueryDslReq.java => QueryS2QLReq.java} | 2 +- .../query/optimizer/MaterializationQuery.java | 16 ++--- .../parser/calcite/CalciteSqlParser.java | 2 +- .../query/parser/calcite/Configuration.java | 4 +- .../parser/calcite/SemanticSchemaManager.java | 18 ++--- .../parser/calcite/planner/AggPlanner.java | 4 +- .../planner/MaterializationPlanner.java | 18 ++--- .../calcite/{dsl => s2ql}/Constants.java | 2 +- .../calcite/{dsl => s2ql}/DataSource.java | 4 +- .../calcite/{dsl => s2ql}/Dimension.java | 2 +- .../DimensionTimeTypeParams.java | 2 +- .../calcite/{dsl => s2ql}/Identify.java | 2 +- .../{dsl => s2ql}/Materialization.java | 2 +- .../{dsl => s2ql}/MaterializationElement.java | 2 +- .../parser/calcite/{dsl => s2ql}/Measure.java | 2 +- .../parser/calcite/{dsl => s2ql}/Metric.java | 2 +- .../{dsl => s2ql}/MetricTypeParams.java | 2 +- .../calcite/{dsl => s2ql}/SemanticModel.java | 2 +- .../calcite/{dsl => s2ql}/TimeRange.java | 2 +- .../parser/calcite/schema/SchemaBuilder.java | 6 +- .../parser/calcite/schema/SemanticSchema.java | 10 +-- .../query/parser/calcite/sql/Renderer.java | 10 +-- ...torImpl.java => S2QLSqlValidatorImpl.java} | 4 +- .../query/parser/calcite/sql/TableView.java | 2 +- .../calcite/sql/node/DataSourceNode.java | 6 +- .../calcite/sql/node/DimensionNode.java | 2 +- .../parser/calcite/sql/node/IdentifyNode.java | 4 +- .../parser/calcite/sql/node/MeasureNode.java | 2 +- .../parser/calcite/sql/node/MetricNode.java | 2 +- .../calcite/sql/render/FilterRender.java | 6 +- .../parser/calcite/sql/render/JoinRender.java | 14 ++-- .../calcite/sql/render/OutputRender.java | 2 +- .../calcite/sql/render/SourceRender.java | 12 ++-- .../parser/convert/QueryReqConverter.java | 16 ++--- .../semantic/query/rest/QueryController.java | 12 ++-- .../semantic/query/service/QueryService.java | 4 +- .../query/service/QueryServiceImpl.java | 14 ++-- .../semantic/query/utils/DimValueAspect.java | 6 +- .../query/utils/QueryStructUtils.java | 16 ++--- ...DslDataAspect.java => S2QLDataAspect.java} | 46 ++++++------- ...ion.java => S2QLPermissionAnnotation.java} | 2 +- .../semantic/query/utils/StatUtils.java | 16 ++--- 77 files changed, 320 insertions(+), 305 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/CommonAgentTool.java rename chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/{DslTool.java => LLMParserTool.java} (66%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl => s2ql}/HeuristicModelResolver.java (99%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl/LLMDslParser.java => s2ql/LLMS2QLParser.java} (89%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl => s2ql}/ModelMatchResult.java (70%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl => s2ql}/ModelResolver.java (83%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl/DSLParseResult.java => s2ql/ParseResult.java} (51%) rename chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/{dsl/DSLDateHelper.java => s2ql/S2QLDateHelper.java} (96%) rename chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/{dsl => s2ql}/LLMReq.java (90%) rename chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/{dsl => s2ql}/LLMResp.java (85%) rename chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/{dsl/DslQuery.java => s2ql/S2QLQuery.java} (76%) rename chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/{dsl/LLMDslParserTest.java => s2ql/LLMS2QLParserTest.java} (93%) rename semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/{QueryDslReq.java => QueryS2QLReq.java} (89%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Constants.java (91%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/DataSource.java (72%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Dimension.java (86%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/DimensionTimeTypeParams.java (67%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Identify.java (81%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Materialization.java (95%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/MaterializationElement.java (74%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Measure.java (85%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/Metric.java (85%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/MetricTypeParams.java (68%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/SemanticModel.java (87%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/{dsl => s2ql}/TimeRange.java (67%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/{DSLSqlValidatorImpl.java => S2QLSqlValidatorImpl.java} (72%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/{DslDataAspect.java => S2QLDataAspect.java} (84%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/{DslPermissionAnnotation.java => S2QLPermissionAnnotation.java} (88%) diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java index c99820095..cfa20665f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -37,7 +37,7 @@ public interface SemanticInterpreter { QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); - QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user); + QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user); QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java index 62a7e993b..6d27baecc 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java @@ -5,7 +5,7 @@ import lombok.Data; @Data public class SqlInfo { - private String llmParseSql; + private String s2QL; private String logicSql; private String querySql; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java index 20500a644..ebe0c732e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.agent.tool; public enum AgentToolType { RULE, - DSL, + LLM_S2QL, PLUGIN, INTERPRET } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/CommonAgentTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/CommonAgentTool.java new file mode 100644 index 000000000..7c90e9be0 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/CommonAgentTool.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.chat.agent.tool; + + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CommonAgentTool extends AgentTool { + + protected List modelIds; + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/DslTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/LLMParserTool.java similarity index 66% rename from chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/DslTool.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/LLMParserTool.java index f4e64a2c1..dd57a1833 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/DslTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/LLMParserTool.java @@ -5,9 +5,7 @@ import lombok.Data; import java.util.List; @Data -public class DslTool extends AgentTool { - - private List modelIds; +public class LLMParserTool extends CommonAgentTool { private List exampleQuestions; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/RuleQueryTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/RuleQueryTool.java index 0da793120..51ec62936 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/RuleQueryTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/RuleQueryTool.java @@ -7,9 +7,8 @@ import org.apache.commons.collections.CollectionUtils; import java.util.List; @Data -public class RuleQueryTool extends AgentTool { +public class RuleQueryTool extends CommonAgentTool { - private List modelIds; private List queryModes; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java index 9217888cf..9b4934e59 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.corrector; import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; -import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; -import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; -import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue; +import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; @@ -67,11 +67,11 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector { return null; } - DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class); - if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) { + ParseResult parseResult = JsonUtil.toObject(JsonUtil.toString(context), ParseResult.class); + if (Objects.isNull(parseResult) || Objects.isNull(parseResult.getLlmReq())) { return null; } - LLMReq llmReq = dslParseResult.getLlmReq(); + LLMReq llmReq = parseResult.getLlmReq(); return llmReq.getLinking(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java index ade49588b..e87c18aec 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper; +import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; @@ -72,7 +72,7 @@ public class WhereCorrector extends BaseSemanticCorrector { String sql = semanticCorrectInfo.getSql(); List whereFields = SqlParserSelectHelper.getWhereFields(sql); if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DateUtils.DATE_FIELD)) { - String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId()); + String currentDate = S2QLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId()); if (StringUtils.isNotBlank(currentDate)) { sql = SqlParserAddHelper.addParenthesisToWhere(sql); sql = SqlParserAddHelper.addWhere(sql, DateUtils.DATE_FIELD, currentDate); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java index a69156a0b..53bdc704b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java @@ -88,7 +88,7 @@ public class MapperHelper { AgentService agentService = ContextUtils.getBean(AgentService.class); - Set detectModelIds = agentService.getDslToolsModelIds(request.getAgentId(), null); + Set detectModelIds = agentService.getModelIds(request.getAgentId(), null); //contains all if (agentService.containsAllModel(detectModelIds)) { if (Objects.nonNull(modelId) && modelId > 0) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java index fe5bffb66..f778a5b98 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.config.OptimizationConfig; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.common.util.ContextUtils; import lombok.extern.slf4j.Slf4j; @@ -20,7 +20,7 @@ public class SatisfactionChecker { // check all the parse info in candidate public static boolean check(QueryContext queryContext) { for (SemanticQuery query : queryContext.getCandidateQueries()) { - if (query.getQueryMode().equals(DslQuery.QUERY_MODE)) { + if (query.getQueryMode().equals(S2QLQuery.QUERY_MODE)) { continue; } if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/HeuristicModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/HeuristicModelResolver.java similarity index 99% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/HeuristicModelResolver.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/HeuristicModelResolver.java index 3fc444062..f93399cc2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/HeuristicModelResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/HeuristicModelResolver.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java similarity index 89% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java index 85c8b6c91..36763b60c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import com.google.common.collect.Lists; import com.tencent.supersonic.chat.agent.tool.AgentToolType; -import com.tencent.supersonic.chat.agent.tool.DslTool; +import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.api.component.SemanticCorrector; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.ChatContext; @@ -18,10 +18,10 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.config.LLMParserConfig; import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.query.QueryManager; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; -import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; -import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue; -import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.utils.ComponentFactory; @@ -59,18 +59,18 @@ import org.springframework.util.CollectionUtils; import org.springframework.web.client.RestTemplate; @Slf4j -public class LLMDslParser implements SemanticParser { +public class LLMS2QLParser implements SemanticParser { @Override public void parse(QueryContext queryCtx, ChatContext chatCtx) { QueryReq request = queryCtx.getRequest(); LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); if (StringUtils.isEmpty(llmParserConfig.getUrl())) { - log.info("llm parser url is empty, skip dsl parser, llmParserConfig:{}", llmParserConfig); + log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2QLParser.class, llmParserConfig); return; } if (SatisfactionChecker.check(queryCtx)) { - log.info("skip dsl parser, queryText:{}", request.getQueryText()); + log.info("skip {}, queryText:{}", LLMS2QLParser.class, request.getQueryText()); return; } try { @@ -79,9 +79,9 @@ public class LLMDslParser implements SemanticParser { return; } - DslTool dslTool = getDslTool(request, modelId); - if (Objects.isNull(dslTool)) { - log.info("no dsl tool in this agent, skip dsl parser"); + CommonAgentTool commonAgentTool = getParserTool(request, modelId); + if (Objects.isNull(commonAgentTool)) { + log.info("no tool in this agent, skip {}", LLMS2QLParser.class); return; } @@ -91,10 +91,10 @@ public class LLMDslParser implements SemanticParser { if (Objects.isNull(llmResp)) { return; } - DSLParseResult dslParseResult = DSLParseResult.builder().request(request) - .dslTool(dslTool).llmReq(llmReq).llmResp(llmResp).build(); + ParseResult parseResult = ParseResult.builder().request(request) + .commonAgentTool(commonAgentTool).llmReq(llmReq).llmResp(llmResp).build(); - SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, dslTool, dslParseResult); + SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, commonAgentTool, parseResult); SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput()); @@ -103,7 +103,7 @@ public class LLMDslParser implements SemanticParser { updateParseInfo(semanticCorrectInfo, modelId, parseInfo); } catch (Exception e) { - log.error("LLMDSLParser error", e); + log.error("LLMS2QLParser error", e); } } @@ -243,12 +243,12 @@ public class LLMDslParser implements SemanticParser { .queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql) .parseInfo(parseInfo).build(); - List dslCorrections = ComponentFactory.getSqlCorrections(); + List corrections = ComponentFactory.getSqlCorrections(); - dslCorrections.forEach(dslCorrection -> { + corrections.forEach(correction -> { try { - dslCorrection.correct(correctInfo); - log.info("sqlCorrection:{} sql:{}", dslCorrection.getClass().getSimpleName(), correctInfo.getSql()); + correction.correct(correctInfo); + log.info("sqlCorrection:{} sql:{}", correction.getClass().getSimpleName(), correctInfo.getSql()); } catch (Exception e) { log.error(String.format("correct error,correctInfo:%s", correctInfo), e); } @@ -256,21 +256,21 @@ public class LLMDslParser implements SemanticParser { return correctInfo; } - private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, DslTool dslTool, - DSLParseResult dslParseResult) { - PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(DslQuery.QUERY_MODE); + private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, CommonAgentTool commonAgentTool, + ParseResult parseResult) { + PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId)); Map properties = new HashMap<>(); - properties.put(Constants.CONTEXT, dslParseResult); + properties.put(Constants.CONTEXT, parseResult); properties.put("type", "internal"); - properties.put("name", dslTool.getName()); + properties.put("name", commonAgentTool.getName()); parseInfo.setProperties(properties); parseInfo.setScore(queryCtx.getRequest().getQueryText().length()); parseInfo.setQueryMode(semanticQuery.getQueryMode()); - parseInfo.getSqlInfo().setLlmParseSql(dslParseResult.getLlmResp().getSqlOutput()); + parseInfo.getSqlInfo().setS2QL(parseResult.getLlmResp().getSqlOutput()); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); Map modelIdToName = semanticSchema.getModelIdToName(); @@ -284,10 +284,11 @@ public class LLMDslParser implements SemanticParser { return parseInfo; } - private DslTool getDslTool(QueryReq request, Long modelId) { + private CommonAgentTool getParserTool(QueryReq request, Long modelId) { AgentService agentService = ContextUtils.getBean(AgentService.class); - List dslTools = agentService.getDslTools(request.getAgentId(), AgentToolType.DSL); - Optional dslToolOptional = dslTools.stream() + List commonAgentTools = agentService.getParserTools(request.getAgentId(), + AgentToolType.LLM_S2QL); + Optional llmParserTool = commonAgentTools.stream() .filter(tool -> { List modelIds = tool.getModelIds(); if (agentService.containsAllModel(new HashSet<>(modelIds))) { @@ -296,18 +297,18 @@ public class LLMDslParser implements SemanticParser { return modelIds.contains(modelId); }) .findFirst(); - return dslToolOptional.orElse(null); + return llmParserTool.orElse(null); } private Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) { AgentService agentService = ContextUtils.getBean(AgentService.class); - Set distinctModelIds = agentService.getDslToolsModelIds(agentId, AgentToolType.DSL); + Set distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2QL); if (agentService.containsAllModel(distinctModelIds)) { distinctModelIds = new HashSet<>(); } ModelResolver modelResolver = ComponentFactory.getModelResolver(); Long modelId = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds); - log.info("resolve modelId:{},dslModels:{}", modelId, distinctModelIds); + log.info("resolve modelId:{},llmParser Models:{}", modelId, distinctModelIds); return modelId; } @@ -354,7 +355,7 @@ public class LLMDslParser implements SemanticParser { linking.addAll(getValueList(queryCtx, modelId, semanticSchema)); llmReq.setLinking(linking); - String currentDate = DSLDateHelper.getReferenceDate(modelId); + String currentDate = S2QLDateHelper.getReferenceDate(modelId); llmReq.setCurrentDate(currentDate); return llmReq; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelMatchResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelMatchResult.java similarity index 70% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelMatchResult.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelMatchResult.java index f85463b4a..1b23f944f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelMatchResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelMatchResult.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelResolver.java similarity index 83% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelResolver.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelResolver.java index 464612ec7..e413853b6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelResolver.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import com.tencent.supersonic.chat.api.pojo.ChatContext; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ParseResult.java similarity index 51% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLParseResult.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ParseResult.java index ab73f04f8..4efc8a351 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLParseResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ParseResult.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; -import com.tencent.supersonic.chat.agent.tool.DslTool; +import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; -import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -13,7 +13,7 @@ import lombok.NoArgsConstructor; @Builder @AllArgsConstructor @NoArgsConstructor -public class DSLParseResult { +public class ParseResult { private LLMReq llmReq; @@ -21,5 +21,5 @@ public class DSLParseResult { private QueryReq request; - private DslTool dslTool; + private CommonAgentTool commonAgentTool; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/S2QLDateHelper.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/S2QLDateHelper.java index c3ae24334..277ba25cd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/S2QLDateHelper.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; @@ -11,7 +11,7 @@ import java.util.List; import java.util.Objects; import org.apache.commons.collections.CollectionUtils; -public class DSLDateHelper { +public class S2QLDateHelper { public static String getReferenceDate(Long modelId) { String defaultDate = DateUtils.getBeforeDate(0); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java index b5972d092..f921f85dd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.plugin.PluginRecallResult; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.common.util.ContextUtils; import java.net.URI; @@ -102,7 +102,7 @@ public class FunctionBasedParser extends PluginParser { log.info("user decide Model:{}", modelId); List plugins = getPluginList(queryContext); List functionDOList = plugins.stream().filter(plugin -> { - if (DslQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) { + if (S2QLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) { return false; } if (plugin.getParseModeConfig() == null) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMReq.java similarity index 90% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMReq.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMReq.java index 1dbebeb61..d8c3ba5ef 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMReq.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.query.llm.dsl; +package com.tencent.supersonic.chat.query.llm.s2ql; import java.util.List; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMResp.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMResp.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMResp.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMResp.java index a7ead1e87..9563a0ee4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMResp.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMResp.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.query.llm.dsl; +package com.tencent.supersonic.chat.query.llm.s2ql; import java.util.List; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/S2QLQuery.java similarity index 76% rename from chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/S2QLQuery.java index 31b67dfba..687a99159 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/S2QLQuery.java @@ -1,10 +1,10 @@ -package com.tencent.supersonic.chat.query.llm.dsl; +package com.tencent.supersonic.chat.query.llm.s2ql; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; +import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.utils.ComponentFactory; @@ -16,7 +16,7 @@ import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,12 +26,12 @@ import org.springframework.stereotype.Component; @Slf4j @Component -public class DslQuery extends PluginSemanticQuery { +public class S2QLQuery extends PluginSemanticQuery { - public static final String QUERY_MODE = "DSL"; + public static final String QUERY_MODE = "LLM_S2QL"; protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - public DslQuery() { + public S2QLQuery() { QueryManager.register(this); } @@ -45,10 +45,10 @@ public class DslQuery extends PluginSemanticQuery { LLMResp llmResp = getLlmResp(); long startTime = System.currentTimeMillis(); - QueryDslReq queryDslReq = getQueryDslReq(llmResp); - QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByDsl(queryDslReq, user); + QueryS2QLReq queryS2QLReq = getQueryS2QLReq(llmResp); + QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user); - log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput()); + log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput()); QueryResult queryResult = new QueryResult(); if (Objects.nonNull(queryResp)) { @@ -69,13 +69,12 @@ public class DslQuery extends PluginSemanticQuery { private LLMResp getLlmResp() { String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); - DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); - return dslParseResult.getLlmResp(); + ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class); + return parseResult.getLlmResp(); } - private QueryDslReq getQueryDslReq(LLMResp llmResp) { - QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId()); - return queryDslReq; + private QueryS2QLReq getQueryS2QLReq(LLMResp llmResp) { + return QueryReqBuilder.buildS2QLReq(llmResp.getCorrectorSql(), parseInfo.getModelId()); } @Override @@ -84,7 +83,7 @@ public class DslQuery extends PluginSemanticQuery { try { explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryTypeEnum.SQL) - .queryReq(getQueryDslReq(getLlmResp())) + .queryReq(getQueryS2QLReq(getLlmResp())) .build(); return semanticInterpreter.explain(explainSqlReq, user); } catch (Exception e) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java index d658aee0f..c5d856a6c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.QueryManager; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; import java.util.List; @@ -24,7 +24,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder { return; } String queryMode = semanticParseInfo.getQueryMode(); - if (QueryManager.isPluginQuery(queryMode) && !DslQuery.QUERY_MODE.equals(queryMode)) { + if (QueryManager.isPluginQuery(queryMode) && !S2QLQuery.QUERY_MODE.equals(queryMode)) { return; } SemanticService semanticService = ContextUtils.getBean(SemanticService.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java index 15a8e041a..9602e008a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.query.QueryManager; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; import java.util.List; @@ -24,7 +24,7 @@ public class EntityInfoParseResponder implements ParseResponder { QueryReq queryReq = queryContext.getRequest(); selectedParses.forEach(parseInfo -> { if (QueryManager.isPluginQuery(parseInfo.getQueryMode()) - && !DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { + && !S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { return; } //1. set entity info diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/AgentService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/AgentService.java index 087c5265e..5b46dcb8f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/AgentService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/AgentService.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.tool.AgentToolType; -import com.tencent.supersonic.chat.agent.tool.DslTool; +import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import java.util.List; import java.util.Set; @@ -19,9 +19,9 @@ public interface AgentService { void deleteAgent(Integer id); - List getDslTools(Integer agentId, AgentToolType agentToolType); + List getParserTools(Integer agentId, AgentToolType agentToolType); - Set getDslToolsModelIds(Integer agentId, AgentToolType agentToolType); + Set getModelIds(Integer agentId, AgentToolType agentToolType); boolean containsAllModel(Set detectModelIds); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/AgentServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/AgentServiceImpl.java index 889776906..d3e3a649f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/AgentServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/AgentServiceImpl.java @@ -5,7 +5,7 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.tool.AgentToolType; -import com.tencent.supersonic.chat.agent.tool.DslTool; +import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.persistence.dataobject.AgentDO; import com.tencent.supersonic.chat.persistence.repository.AgentRepository; import com.tencent.supersonic.chat.service.AgentService; @@ -87,7 +87,7 @@ public class AgentServiceImpl implements AgentService { return agentDO; } - public List getDslTools(Integer agentId, AgentToolType agentToolType) { + public List getParserTools(Integer agentId, AgentToolType agentToolType) { Agent agent = getAgent(agentId); if (agent == null) { return Lists.newArrayList(); @@ -96,15 +96,16 @@ public class AgentServiceImpl implements AgentService { if (CollectionUtils.isEmpty(tools)) { return Lists.newArrayList(); } - return tools.stream().map(tool -> JSONObject.parseObject(tool, DslTool.class)).collect(Collectors.toList()); + return tools.stream().map(tool -> JSONObject.parseObject(tool, CommonAgentTool.class)) + .collect(Collectors.toList()); } - public Set getDslToolsModelIds(Integer agentId, AgentToolType agentToolType) { - List dslTools = getDslTools(agentId, agentToolType); - if (CollectionUtils.isEmpty(dslTools)) { + public Set getModelIds(Integer agentId, AgentToolType agentToolType) { + List commonAgentTools = getParserTools(agentId, agentToolType); + if (CollectionUtils.isEmpty(commonAgentTools)) { return new HashSet<>(); } - return dslTools.stream().map(DslTool::getModelIds) + return commonAgentTools.stream().map(CommonAgentTool::getModelIds) .filter(modelIds -> !CollectionUtils.isEmpty(modelIds)) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index e339a1ad3..11878653e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -19,15 +19,15 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; +import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.CostType; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QuerySelector; -import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; -import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; +import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.service.ChatService; @@ -239,7 +239,7 @@ public class QueryServiceImpl implements QueryService { } private void saveSolvedQuery(ExecuteQueryReq queryReq, SemanticParseInfo parseInfo, - ChatQueryDO chatQueryDO, QueryResult queryResult) { + ChatQueryDO chatQueryDO, QueryResult queryResult) { if (queryResult.getResponse() == null && CollectionUtils.isEmpty(queryResult.getQueryResults())) { return; } @@ -305,12 +305,12 @@ public class QueryServiceImpl implements QueryService { SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); - if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { + if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { Map> filedNameToValueMap = new HashMap<>(); Map> havingFiledNameToValueMap = new HashMap<>(); String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); - DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); - LLMResp llmResp = dslParseResult.getLlmResp(); + ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class); + LLMResp llmResp = parseResult.getLlmResp(); String correctorSql = llmResp.getCorrectorSql(); log.info("correctorSql before replacing:{}", correctorSql); @@ -330,9 +330,9 @@ public class QueryServiceImpl implements QueryService { correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap); log.info("correctorSql after replacing:{}", correctorSql); llmResp.setCorrectorSql(correctorSql); - dslParseResult.setLlmResp(llmResp); + parseResult.setLlmResp(llmResp); Map properties = new HashMap<>(); - properties.put(Constants.CONTEXT, dslParseResult); + properties.put(Constants.CONTEXT, parseResult); parseInfo.setProperties(properties); parseInfo.getSqlInfo().setLogicSql(correctorSql); semanticQuery.setParseInfo(parseInfo); @@ -399,29 +399,29 @@ public class QueryServiceImpl implements QueryService { if (CollectionUtils.isEmpty(metricFilters)) { return; } - for (QueryFilter dslQueryFilter : metricFilters) { + for (QueryFilter queryFilter : metricFilters) { Map map = new HashMap<>(); for (FilterExpression filterExpression : filterExpressionList) { if (filterExpression.getFieldName() != null - && filterExpression.getFieldName().contains(dslQueryFilter.getName()) - && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { - map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); + && filterExpression.getFieldName().contains(queryFilter.getName()) + && queryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { + map.put(filterExpression.getFieldValue().toString(), queryFilter.getValue().toString()); contextMetricFilters.stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); + if (o.getName().equals(queryFilter.getName())) { + o.setValue(queryFilter.getValue()); } }); break; } } - filedNameToValueMap.put(dslQueryFilter.getName(), map); + filedNameToValueMap.put(queryFilter.getName(), map); } } private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) { SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); - if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { + if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { return parseInfo; } if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java index 489a43b30..64478adec 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver; +import com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver; import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder; @@ -20,7 +20,7 @@ public class ComponentFactory { private static List schemaMappers = new ArrayList<>(); private static List semanticParsers = new ArrayList<>(); - private static List dslCorrections = new ArrayList<>(); + private static List s2QLCorrections = new ArrayList<>(); private static SemanticInterpreter semanticInterpreter; private static List parseResponders = new ArrayList<>(); private static List executeResponders = new ArrayList<>(); @@ -35,7 +35,8 @@ public class ComponentFactory { } public static List getSqlCorrections() { - return CollectionUtils.isEmpty(dslCorrections) ? init(SemanticCorrector.class, dslCorrections) : dslCorrections; + return CollectionUtils.isEmpty(s2QLCorrections) ? init(SemanticCorrector.class, + s2QLCorrections) : s2QLCorrections; } public static List getParseResponders() { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java index 6b955b7ac..6f61dc241 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java @@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import java.time.LocalDate; @@ -123,19 +123,19 @@ public class QueryReqBuilder { } /** - * convert to QueryDslReq + * convert to QueryS2QLReq * * @param querySql * @param modelId * @return */ - public static QueryDslReq buildDslReq(String querySql, Long modelId) { - QueryDslReq queryDslReq = new QueryDslReq(); + public static QueryS2QLReq buildS2QLReq(String querySql, Long modelId) { + QueryS2QLReq queryS2QLReq = new QueryS2QLReq(); if (Objects.nonNull(querySql)) { - queryDslReq.setSql(querySql); + queryS2QLReq.setSql(querySql); } - queryDslReq.setModelId(modelId); - return queryDslReq; + queryS2QLReq.setModelId(modelId); + return queryS2QLReq; } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParserTest.java similarity index 93% rename from chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java rename to chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParserTest.java index 7c233c448..869efa136 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParserTest.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.llm.dsl; +package com.tencent.supersonic.chat.parser.llm.s2ql; import static org.mockito.Mockito.when; @@ -16,7 +16,7 @@ import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; -class LLMDslParserTest { +class LLMS2QLParserTest { @Test void setFilter() { @@ -72,9 +72,9 @@ class LLMDslParserTest { .parseInfo(parseInfo) .build(); - LLMDslParser llmDslParser = new LLMDslParser(); + LLMS2QLParser llms2QLParser = new LLMS2QLParser(); - llmDslParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo); + llms2QLParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo); } } \ No newline at end of file diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java index 00b0a5e42..fabbc8f39 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; @@ -56,9 +56,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @Override @SneakyThrows - public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) { + public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) { queryService = ContextUtils.getBean(QueryService.class); - Object object = queryService.queryBySql(queryDslReq, user); + Object object = queryService.queryBySql(queryS2QLReq, user); QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class); return queryResultWithSchemaResp; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java index 9a28cffe6..2fdbb4f86 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java @@ -31,7 +31,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import java.net.URI; @@ -83,10 +83,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) { + public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), - new Gson().toJson(queryDslReq)); + new Gson().toJson(queryS2QLReq)); } public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) { diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 8117e1a81..dee362afd 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ - com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ + com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \ com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser @@ -21,8 +21,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\ com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.HeuristicQuerySelector -com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\ - com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver +com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\ + com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index 544e02ee0..7e6645b81 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.AgentConfig; import com.tencent.supersonic.chat.agent.tool.AgentToolType; -import com.tencent.supersonic.chat.agent.tool.DslTool; +import com.tencent.supersonic.chat.agent.tool.LLMParserTool; import com.tencent.supersonic.chat.agent.tool.RuleQueryTool; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -228,11 +228,11 @@ public class ConfigureDemo implements ApplicationListener )); agentConfig.getTools().add(ruleQueryTool); - DslTool dslTool = new DslTool(); - dslTool.setId("1"); - dslTool.setType(AgentToolType.DSL); - dslTool.setModelIds(Lists.newArrayList(-1L)); - agentConfig.getTools().add(dslTool); + LLMParserTool llmParserTool = new LLMParserTool(); + llmParserTool.setId("1"); + llmParserTool.setType(AgentToolType.LLM_S2QL); + llmParserTool.setModelIds(Lists.newArrayList(-1L)); + agentConfig.getTools().add(llmParserTool); agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); agentService.createAgent(agent, User.getFakeUser()); @@ -255,11 +255,11 @@ public class ConfigureDemo implements ApplicationListener "ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID")); agentConfig.getTools().add(ruleQueryTool); - DslTool dslTool = new DslTool(); - dslTool.setId("1"); - dslTool.setType(AgentToolType.DSL); - dslTool.setModelIds(Lists.newArrayList(-1L)); - agentConfig.getTools().add(dslTool); + LLMParserTool llmParserTool = new LLMParserTool(); + llmParserTool.setId("1"); + llmParserTool.setType(AgentToolType.LLM_S2QL); + llmParserTool.setModelIds(Lists.newArrayList(-1L)); + agentConfig.getTools().add(llmParserTool); agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); agentService.createAgent(agent, User.getFakeUser()); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 54ce3be22..1f71cbdfd 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ - com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ + com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \ com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \ com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser @@ -22,8 +22,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\ com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.HeuristicQuerySelector -com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\ - com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver +com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\ + com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 5132f22cc..9b45c40e0 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -66,3 +66,5 @@ alter table s2_dimension add column data_type varchar(50) not null DEFAULT 'va alter table s2_query_stat_info add column `query_opt_mode` varchar(20) DEFAULT NULL COMMENT '优化模式'; alter table s2_datasource add column depends text COMMENT '上游依赖标识' after datasource_detail; +--20231018 +UPDATE `s2_agent` SET `config` = replace (`config`,'DSL','LLM_S2QL') WHERE `config` LIKE '%DSL%'; diff --git a/pom.xml b/pom.xml index d8386dfe5..c3d1921db 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 3.2.4 4.5.1 4.5 - 0.7.5-SNAPSHOT + 0.7.6-SNAPSHOT 2.30.0 diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDslReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2QLReq.java similarity index 89% rename from semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDslReq.java rename to semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2QLReq.java index 6c74bbbd7..38979f0da 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDslReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2QLReq.java @@ -6,7 +6,7 @@ import lombok.ToString; @Data @ToString -public class QueryDslReq { +public class QueryS2QLReq { private Long modelId; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java index ce54a1dd2..8ec892319 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java @@ -18,16 +18,16 @@ import com.tencent.supersonic.semantic.materialization.domain.MaterializationCon import com.tencent.supersonic.semantic.materialization.domain.MaterializationRecordService; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java index 7ead020c9..387ebe004 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.parser.SqlParser; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import org.springframework.stereotype.Component; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java index 4d3df0914..a234ad202 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,7 +32,6 @@ import org.apache.calcite.sql2rel.SqlToRelConverter; public class Configuration { public static Properties configProperties = new Properties(); - public static DSLSqlValidatorImpl dslSqlValidator; public static RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties); @@ -71,7 +69,7 @@ public class Configuration { public static SqlValidator getSqlValidator(CalciteSchema rootSchema) { List tables = new ArrayList<>(); tables.add(SqlStdOperatorTable.instance()); - SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); //.of(SqlStdOperatorTable.instance()); + SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); //operatorTable. SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT .withLenientOperatorLookup(config.lenientOperatorLookup()) diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java index adb02101d..19baa8020 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java @@ -11,15 +11,15 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.model.domain.Catalog; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MetricTypeParams; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DimensionTimeTypeParams; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MetricTypeParams; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import java.util.ArrayList; import java.util.HashMap; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java index 931692e20..819b6de6d 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner; import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java index 74f1dad63..790d0a91f 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java @@ -4,15 +4,15 @@ import com.tencent.supersonic.common.util.calcite.SqlParseUtils; import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Constants.java similarity index 91% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Constants.java index 4757cde33..56c7af844 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Constants.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; public class Constants { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataSource.java similarity index 72% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataSource.java index e9efd3fa3..cae655257 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataSource.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; import java.util.List; import lombok.Builder; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Dimension.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Dimension.java similarity index 86% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Dimension.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Dimension.java index 3d23e6e7e..9fa8114ef 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Dimension.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Dimension.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DimensionTimeTypeParams.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DimensionTimeTypeParams.java similarity index 67% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DimensionTimeTypeParams.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DimensionTimeTypeParams.java index 0ff1cc48b..25e3c2c5b 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DimensionTimeTypeParams.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DimensionTimeTypeParams.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Identify.java similarity index 81% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Identify.java index 61458caca..9c09f5075 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Identify.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Materialization.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Materialization.java similarity index 95% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Materialization.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Materialization.java index f0675d6a0..e2ebc898b 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Materialization.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Materialization.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import java.util.ArrayList; import java.util.List; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MaterializationElement.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MaterializationElement.java similarity index 74% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MaterializationElement.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MaterializationElement.java index 8af50780f..e9aff0b94 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MaterializationElement.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MaterializationElement.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import java.util.List; import lombok.Builder; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Measure.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Measure.java similarity index 85% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Measure.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Measure.java index 1d8595595..f80378fed 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Measure.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Measure.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Metric.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Metric.java similarity index 85% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Metric.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Metric.java index 2683f940a..c092ac93c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Metric.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Metric.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MetricTypeParams.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MetricTypeParams.java similarity index 68% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MetricTypeParams.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MetricTypeParams.java index 7e2a60a71..51934b265 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MetricTypeParams.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MetricTypeParams.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import java.util.List; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/SemanticModel.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/SemanticModel.java similarity index 87% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/SemanticModel.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/SemanticModel.java index 16f100fda..144be1a8e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/SemanticModel.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/SemanticModel.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import java.util.ArrayList; import java.util.HashMap; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/TimeRange.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/TimeRange.java similarity index 67% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/TimeRange.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/TimeRange.java index 580d203a8..eb2aad7f0 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/TimeRange.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/TimeRange.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.parser.calcite.dsl; +package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; import lombok.Builder; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java index 4c32d526d..75705a40c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2QLSqlValidatorImpl; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -35,9 +35,9 @@ public class SchemaBuilder { Configuration.typeFactory, Configuration.config ); - DSLSqlValidatorImpl dslSqlValidator = new DSLSqlValidatorImpl(Configuration.operatorTable, catalogReader, + S2QLSqlValidatorImpl s2QLSqlValidator = new S2QLSqlValidatorImpl(Configuration.operatorTable, catalogReader, Configuration.typeFactory, Configuration.validatorConfig); - return new ParameterScope(dslSqlValidator, nameToTypeMap); + return new ParameterScope(s2QLSqlValidator, nameToTypeMap); } public static CalciteSchema getMaterializationSchema() { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java index a69666872..2e3de1672 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java index dcaf97a1c..57b77e846 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java @@ -5,11 +5,11 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import java.util.HashSet; import java.util.List; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/DSLSqlValidatorImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2QLSqlValidatorImpl.java similarity index 72% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/DSLSqlValidatorImpl.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2QLSqlValidatorImpl.java index d3a07d9be..662bc9c65 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/DSLSqlValidatorImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2QLSqlValidatorImpl.java @@ -5,9 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorImpl; -public class DSLSqlValidatorImpl extends SqlValidatorImpl { +public class S2QLSqlValidatorImpl extends SqlValidatorImpl { - public DSLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader, + public S2QLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader, RelDataTypeFactory typeFactory, Config config) { super(opTab, catalogReader, typeFactory, config); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java index a3f0a2eb7..151e4661f 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java index d2edf3f88..53f47d705 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java @@ -3,9 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import java.util.ArrayList; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java index 394e56e35..e1afce4e3 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import java.util.List; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java index 1203ad739..0765f5ac0 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type; import java.util.List; import java.util.Optional; import java.util.Set; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java index 777df9321..15843133f 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java index 081f35048..2c92a7ac0 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import java.util.HashMap; import java.util.Map; import lombok.Data; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java index ea34771f7..492d46844 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java @@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import java.util.ArrayList; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java index 09dd852ae..2539d2b57 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java @@ -1,15 +1,15 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render; import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java index bc1562cf4..e6fc0b363 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import java.util.ArrayList; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java index db5c399dd..b2cfa6c23 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java @@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render; import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; -import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; +import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java index 537bcae63..a91b0542a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java @@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.ModelService; @@ -54,7 +54,7 @@ public class QueryReqConverter { @Autowired private Catalog catalog; - public QueryStatement convert(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception { + public QueryStatement convert(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception { if (Objects.isNull(modelSchemaResp)) { return new QueryStatement(); @@ -117,9 +117,9 @@ public class QueryReqConverter { return queryStatement; } - private AggOption getAggOption(QueryDslReq databaseReq) { - // if there is no group by in dsl,set MetricTable's aggOption to "NATIVE" - // if there is count() in dsl,set MetricTable's aggOption to "NATIVE" + private AggOption getAggOption(QueryS2QLReq databaseReq) { + // if there is no group by in S2QL,set MetricTable's aggOption to "NATIVE" + // if there is count() in S2QL,set MetricTable's aggOption to "NATIVE" String sql = databaseReq.getSql(); if (!SqlParserSelectHelper.hasGroupBy(sql) || SqlParserSelectFunctionHelper.hasFunction(sql, "count")) { @@ -128,7 +128,7 @@ public class QueryReqConverter { return AggOption.DEFAULT; } - private void convertNameToBizName(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) { + private void convertNameToBizName(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) { Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); @@ -155,7 +155,7 @@ public class QueryReqConverter { return metrics; } - private void functionNameCorrector(QueryDslReq databaseReq) { + private void functionNameCorrector(QueryS2QLReq databaseReq) { DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); if (Objects.isNull(database) || Objects.isNull(database.getType())) { return; @@ -182,7 +182,7 @@ public class QueryReqConverter { return result; } - public void correctTableName(QueryDslReq databaseReq) { + public void correctTableName(QueryS2QLReq databaseReq) { String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId()); databaseReq.setSql(sql); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java index 97564dd7a..9e0c2cdc4 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; @@ -42,11 +42,11 @@ public class QueryController { @PostMapping("/sql") - public Object queryBySql(@RequestBody QueryDslReq queryDslReq, + public Object queryBySql(@RequestBody QueryS2QLReq queryS2QLReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - Object queryBySql = queryService.queryBySql(queryDslReq, user); + Object queryBySql = queryService.queryBySql(queryS2QLReq, user); log.info("queryBySql:{},queryBySql"); return queryBySql; } @@ -124,9 +124,9 @@ public class QueryController { QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); if (QueryTypeEnum.SQL.equals(queryTypeEnum)) { - QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class); - ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() - .queryReq(queryDslReq) + QueryS2QLReq queryS2QLReq = JsonUtil.toObject(queryReqJson, QueryS2QLReq.class); + ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() + .queryReq(queryS2QLReq) .queryTypeEnum(queryTypeEnum).build(); return queryService.explain(explainSqlReqNew, user); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java index 8685d8652..46058f785 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; @@ -18,7 +18,7 @@ import java.util.List; public interface QueryService { - Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception; + Object queryBySql(QueryS2QLReq querySqlCmd, User user) throws Exception; QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index ac8525bac..c16d49d1f 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -24,11 +24,11 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; -import com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation; +import com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation; import com.tencent.supersonic.semantic.query.executor.QueryExecutor; import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; @@ -89,9 +89,9 @@ public class QueryServiceImpl implements QueryService { } @Override - @DslPermissionAnnotation + @S2QLPermissionAnnotation @SneakyThrows - public Object queryBySql(QueryDslReq querySqlCmd, User user) { + public Object queryBySql(QueryS2QLReq querySqlCmd, User user) { statUtils.initStatInfo(querySqlCmd, user); QueryStatement queryStatement = new QueryStatement(); try { @@ -109,7 +109,7 @@ public class QueryServiceImpl implements QueryService { return semanticQueryEngine.execute(queryStatement); } - private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception { + private QueryStatement convertToQueryStatement(QueryS2QLReq querySqlCmd, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); List modelIds = new ArrayList<>(); modelIds.add(querySqlCmd.getModelId()); @@ -294,8 +294,8 @@ public class QueryServiceImpl implements QueryService { QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); T queryReq = explainSqlReq.getQueryReq(); - if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) { - QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user); + if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2QLReq) { + QueryStatement queryStatement = convertToQueryStatement((QueryS2QLReq) queryReq, user); return getExplainResp(queryStatement); } if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java index 6393f7102..740db6c84 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.pojo.Filter; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; import java.util.ArrayList; @@ -45,9 +45,9 @@ public class DimValueAspect { return queryResultWithColumns; } Object[] args = joinPoint.getArgs(); - QueryDslReq queryDslReq = (QueryDslReq) args[0]; + QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0]; - List dimensions = dimensionService.getDimensions(queryDslReq.getModelId()); + List dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId()); Map> techNameToBizName = getTechNameToBizName(dimensions); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java index 5ac386c1f..1576226aa 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java @@ -24,7 +24,7 @@ import com.tencent.supersonic.semantic.api.model.response.ItemDateResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; @@ -188,8 +188,8 @@ public class QueryStructUtils { return resNameEnSet; } - public Set getResName(QueryDslReq queryDslReq) { - Set resNameSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql()) + public Set getResName(QueryS2QLReq queryS2QLReq) { + Set resNameSet = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql()) .stream().collect(Collectors.toSet()); return resNameSet; } @@ -199,11 +199,11 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getResNameEnExceptInternalCol(QueryDslReq queryDslReq, User user) { - Set resNameSet = getResName(queryDslReq); + public Set getResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq, User user) { + Set resNameSet = getResName(queryS2QLReq); Set resNameEnSet = new HashSet<>(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - List modelIds = Lists.newArrayList(queryDslReq.getModelId()); + List modelIds = Lists.newArrayList(queryS2QLReq.getModelId()); filter.setModelIds(modelIds); List modelSchemaRespList = schemaService.fetchModelSchema(filter, user); if (!CollectionUtils.isEmpty(modelSchemaRespList)) { @@ -234,8 +234,8 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getFilterResNameEnExceptInternalCol(QueryDslReq queryDslReq) { - String sql = queryDslReq.getSql(); + public Set getFilterResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq) { + String sql = queryS2QLReq.getSql(); Set resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslDataAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java similarity index 84% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslDataAspect.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java index eaa70367e..d3c145d6c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslDataAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.query.service.AuthCommonService; @@ -41,7 +41,7 @@ import org.springframework.util.CollectionUtils; @Aspect @Order(1) @Slf4j -public class DslDataAspect { +public class S2QLDataAspect { @Autowired private QueryStructUtils queryStructUtils; @@ -54,24 +54,24 @@ public class DslDataAspect { @Value("${permission.data.enable:true}") private Boolean permissionDataEnable; - @Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation)") - private void dslPermissionCheck() { + @Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation)") + private void s2QLPermissionCheck() { } - @Around("dslPermissionCheck()") + @Around("s2QLPermissionCheck()") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { - log.info("dsl permission check!"); + log.info("s2ql permission check!"); Object[] objects = joinPoint.getArgs(); - QueryDslReq queryDslReq = (QueryDslReq) objects[0]; + QueryS2QLReq queryS2QLReq = (QueryS2QLReq) objects[0]; User user = (User) objects[1]; if (!permissionDataEnable) { - log.info("not to check dsl permission!"); + log.info("not to check s2QL permission!"); return joinPoint.proceed(); } if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { throw new RuntimeException("please provide user information"); } - Long modelId = queryDslReq.getModelId(); + Long modelId = queryS2QLReq.getModelId(); //1. determine whether admin of the model if (authCommonService.doModelAdmin(user, modelId)) { @@ -82,7 +82,7 @@ public class DslDataAspect { authCommonService.doModelVisible(user, modelId); // 3. fetch data permission meta information - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryDslReq, user); + Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user); log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege); Set sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId); @@ -97,10 +97,10 @@ public class DslDataAspect { Set resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId); // 4.if sensitive fields without permission are involved in filter, thrown an exception - doFilterCheckLogic(queryDslReq, resAuthSet, sensitiveResReq); + doFilterCheckLogic(queryS2QLReq, resAuthSet, sensitiveResReq); // 5.row permission pre-filter - doRowPermission(queryDslReq, authorizedResource); + doRowPermission(queryS2QLReq, authorizedResource); // 6.proceed QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); @@ -123,7 +123,7 @@ public class DslDataAspect { return queryResultAfterDesensitization; } - private void doRowPermission(QueryDslReq queryDslReq, AuthorizedResourceResp authorizedResource) { + private void doRowPermission(QueryS2QLReq queryS2QLReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); List dimensionFilters = new ArrayList<>(); @@ -143,12 +143,12 @@ public class DslDataAspect { } }); try { - Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner.toString() + " ) "); + Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { - String sql = SqlParserAddHelper.addWhere(queryDslReq.getSql(), expression); - log.info("before doRowPermission, queryDslReq:{}", queryDslReq.getSql()); - queryDslReq.setSql(sql); - log.info("after doRowPermission, queryDslReq:{}", queryDslReq.getSql()); + String sql = SqlParserAddHelper.addWhere(queryS2QLReq.getSql(), expression); + log.info("before doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql()); + queryS2QLReq.setSql(sql); + log.info("after doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql()); } } catch (JSQLParserException jsqlParserException) { log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); @@ -156,22 +156,22 @@ public class DslDataAspect { } - private void doFilterCheckLogic(QueryDslReq queryDslReq, Set resAuthName, - Set sensitiveResReq) { - Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryDslReq); + private void doFilterCheckLogic(QueryS2QLReq queryS2QLReq, Set resAuthName, + Set sensitiveResReq) { + Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2QLReq); Set need2Apply = resFilterSet.stream() .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); Set nameCnSet = new HashSet<>(); List modelIds = new ArrayList<>(); - modelIds.add(queryDslReq.getModelId()); + modelIds.add(queryS2QLReq.getModelId()); List modelInfos = modelService.getModelList(modelIds); String modelNameCn = Constants.EMPTY; if (!CollectionUtils.isEmpty(modelInfos)) { modelNameCn = modelInfos.get(0).getName(); } - List dimensionDescList = dimensionService.getDimensions(queryDslReq.getModelId()); + List dimensionDescList = dimensionService.getDimensions(queryS2QLReq.getModelId()); String finalDomainNameCn = modelNameCn; dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName())) .forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName())); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslPermissionAnnotation.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLPermissionAnnotation.java similarity index 88% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslPermissionAnnotation.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLPermissionAnnotation.java index 8a9c368dd..989b8618b 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslPermissionAnnotation.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLPermissionAnnotation.java @@ -9,6 +9,6 @@ import java.lang.annotation.Documented; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) @Documented -public @interface DslPermissionAnnotation { +public @interface S2QLPermissionAnnotation { } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java index 30f8474ed..74a41fc54 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java @@ -12,7 +12,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.QueryStat; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; -import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.model.domain.ModelService; @@ -87,11 +87,11 @@ public class StatUtils { } - public void initStatInfo(QueryDslReq queryDslReq, User facadeUser) { + public void initStatInfo(QueryS2QLReq queryS2QLReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); - List allFields = SqlParserSelectHelper.getAllFields(queryDslReq.getSql()); - queryStatInfo.setModelId(queryDslReq.getModelId()); - ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryDslReq.getModelId()); + List allFields = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql()); + queryStatInfo.setModelId(queryS2QLReq.getModelId()); + ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2QLReq.getModelId()); List dimensions = new ArrayList<>(); if (Objects.nonNull(modelSchemaResp)) { @@ -106,12 +106,12 @@ public class StatUtils { String userName = getUserName(facadeUser); try { queryStatInfo.setTraceId("") - .setModelId(queryDslReq.getModelId()) + .setModelId(queryS2QLReq.getModelId()) .setUser(userName) .setQueryType(QueryTypeEnum.SQL.getValue()) .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState()) - .setQuerySqlCmd(queryDslReq.toString()) - .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryDslReq.toString())) + .setQuerySqlCmd(queryS2QLReq.toString()) + .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2QLReq.toString())) .setStartTime(System.currentTimeMillis()) .setUseResultCache(true) .setUseSqlCache(true)