From 24b442baef2e7cf1ac54f0111d54297c56b2e192 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Tue, 30 Jan 2024 20:43:53 +0800 Subject: [PATCH] (improvement)(headless)(chat) Add views and adapt chat and headless (#700) * (improvement)(headless)(chat) Add views and adapt chat and headless --------- Co-authored-by: jolunoluo --- .../chat/api/pojo/SchemaElement.java | 6 +- .../chat/api/pojo/SchemaElementType.java | 2 +- .../chat/api/pojo/SchemaMapInfo.java | 24 +- .../api/pojo/SchemaModelClusterMapInfo.java | 61 -- .../chat/api/pojo/SemanticParseInfo.java | 41 +- .../chat/api/pojo/SemanticSchema.java | 89 ++- .../{ModelSchema.java => ViewSchema.java} | 55 +- .../pojo/request/ChatDefaultConfigReq.java | 22 - .../chat/api/pojo/request/PluginQueryReq.java | 2 +- .../api/pojo/request/SimilarQueryReq.java | 2 +- .../response/ChatDefaultRichConfigResp.java | 4 +- .../chat/api/pojo/response/DataInfo.java | 4 + .../chat/api/pojo/response/EntityInfo.java | 6 +- .../{ModelInfo.java => ViewInfo.java} | 2 +- .../supersonic/chat/core/agent/Agent.java | 13 +- .../chat/core/agent/DataAnalyticsTool.java | 11 - .../chat/core/agent/NL2SQLTool.java | 5 +- .../chat/core/agent/RuleParserTool.java | 2 +- .../core/corrector/BaseSemanticCorrector.java | 24 +- .../chat/core/corrector/FromCorrector.java | 21 - .../chat/core/corrector/GroupByCorrector.java | 12 +- .../chat/core/corrector/HavingCorrector.java | 13 +- .../chat/core/corrector/SchemaCorrector.java | 9 +- .../chat/core/corrector/WhereCorrector.java | 20 +- .../core/knowledge/LoadRemoveService.java | 11 +- .../chat/core/knowledge/ModelInfoStat.java | 21 - .../chat/core/knowledge/ViewInfoStat.java | 22 + .../builder/DimensionWordBuilder.java | 11 +- .../knowledge/builder/EntityWordBuilder.java | 9 +- .../knowledge/builder/MetricWordBuilder.java | 11 +- .../knowledge/builder/ModelWordBuilder.java | 7 +- .../knowledge/builder/ValueWordBuilder.java | 9 +- .../knowledge/builder/WordBuilderFactory.java | 2 +- .../semantic/BaseSemanticInterpreter.java | 59 +- .../semantic/LocalSemanticInterpreter.java | 35 +- .../semantic/RemoteSemanticInterpreter.java | 97 +-- .../semantic/SemanticInterpreter.java | 34 +- ...emaBuilder.java => ViewSchemaBuilder.java} | 42 +- .../chat/core/mapper/BaseMapper.java | 25 +- .../chat/core/mapper/BaseMatchStrategy.java | 35 +- .../core/mapper/DatabaseMatchStrategy.java | 21 +- .../chat/core/mapper/EntityMapper.java | 19 +- .../chat/core/mapper/KeywordMapper.java | 21 +- .../chat/core/mapper/MapperHelper.java | 35 +- .../chat/core/mapper/ModelClusterMapper.java | 52 -- .../chat/core/mapper/QueryFilterMapper.java | 19 +- .../chat/core/parser/JavaLLMProxy.java | 15 +- .../supersonic/chat/core/parser/LLMProxy.java | 2 +- .../chat/core/parser/PythonLLMProxy.java | 17 +- .../chat/core/parser/QueryTypeParser.java | 15 +- .../chat/core/parser/plugin/PluginParser.java | 20 +- .../embedding/EmbeddingRecallParser.java | 6 +- .../plugin/function/FunctionCallParser.java | 17 +- .../sql/llm/HeuristicModelResolver.java | 146 ---- .../parser/sql/llm/HeuristicViewResolver.java | 136 ++++ .../parser/sql/llm/LLMRequestService.java | 136 ++-- .../parser/sql/llm/LLMResponseService.java | 6 +- .../core/parser/sql/llm/LLMSqlParser.java | 22 +- .../core/parser/sql/llm/ModelResolver.java | 12 - .../sql/llm/OnePassSCSqlGeneration.java | 15 +- .../parser/sql/llm/OnePassSqlGeneration.java | 17 +- .../chat/core/parser/sql/llm/ParseResult.java | 3 +- .../core/parser/sql/llm/SqlGeneration.java | 4 +- .../parser/sql/llm/SqlPromptGenerator.java | 9 +- .../sql/llm/TwoPassSCSqlGeneration.java | 13 +- .../parser/sql/llm/TwoPassSqlGeneration.java | 11 +- ...lMatchResult.java => ViewMatchResult.java} | 2 +- .../core/parser/sql/llm/ViewResolver.java | 12 + .../parser/sql/rule/AgentCheckParser.java | 13 +- .../parser/sql/rule/ContextInheritParser.java | 47 +- .../core/parser/sql/rule/RuleSqlParser.java | 8 +- .../supersonic/chat/core/plugin/Plugin.java | 4 +- .../chat/core/plugin/PluginManager.java | 17 +- .../chat/core/plugin/PluginRecallResult.java | 3 +- .../chat/core/pojo/QueryContext.java | 13 +- .../chat/core/query/BaseSemanticQuery.java | 26 +- .../chat/core/query/llm/s2sql/LLMReq.java | 5 +- .../core/query/llm/s2sql/LLMSqlQuery.java | 4 +- .../query/plugin/PluginSemanticQuery.java | 2 +- .../chat/core/query/rule/QueryMatcher.java | 7 +- .../core/query/rule/RuleSemanticQuery.java | 23 +- .../query/rule/metric/MetricModelQuery.java | 9 +- .../rule/metric/MetricSemanticQuery.java | 73 +- .../core/query/rule/tag/TagListQuery.java | 51 +- .../core/query/rule/tag/TagSemanticQuery.java | 45 +- .../chat/core/utils/ComponentFactory.java | 8 +- .../chat/core/utils/DictQueryHelper.java | 33 +- .../chat/core/utils/ModelClusterBuilder.java | 42 - .../chat/core/utils/NatureHelper.java | 51 +- .../chat/core/utils/QueryReqBuilder.java | 49 +- .../chat/core/utils/SimilarQueryManager.java | 2 +- .../chat/core/s2sql/LLMSqlParserTest.java | 15 +- .../listener/SchemaDictUpdateListener.java | 4 +- .../persistence/dataobject/PluginDO.java | 234 +----- .../persistence/mapper/PluginDOMapper.java | 63 +- .../repository/PluginRepository.java | 4 +- .../repository/impl/PluginRepositoryImpl.java | 14 +- .../execute/DimensionRecommendProcessor.java | 9 +- .../execute/MetricRecommendProcessor.java | 11 +- .../processor/parse/EntityInfoProcessor.java | 7 +- .../processor/parse/ParseInfoProcessor.java | 39 +- .../server/rest/ChatConfigController.java | 32 +- .../chat/server/service/ChatService.java | 3 +- .../chat/server/service/SemanticService.java | 210 ++--- .../server/service/impl/ChatServiceImpl.java | 7 +- .../service/impl/ConfigServiceImpl.java | 34 +- .../service/impl/PluginServiceImpl.java | 62 +- .../server/service/impl/QueryServiceImpl.java | 34 +- .../service/impl/RecommendServiceImpl.java | 17 +- .../server/service/impl/SchemaService.java | 13 +- .../service/impl/SearchServiceImpl.java | 47 +- .../chat/server/service/impl/WordService.java | 11 +- .../chat/server/util/ChatConfigHelper.java | 6 +- .../chat/server/util/DictMetaHelper.java | 37 +- .../main/resources/mapper/PluginDOMapper.xml | 276 ------- .../processor/MetricCheckProcessorTest.java | 6 +- .../server/utils/QueryReqBuilderTest.java | 21 +- .../supersonic/common/pojo/ModelCluster.java | 60 -- .../common/pojo/enums/DictWordType.java | 4 +- .../common/pojo/enums/TimeMode.java | 11 + .../common/pojo/enums/TypeEnums.java | 33 +- .../common/util/SignatureUtils.java | 40 +- .../headless/api/pojo/AppConfig.java | 11 - .../headless/api/pojo/DrillDownDimension.java | 8 + .../api/pojo/MetricTypeDefaultConfig.java | 10 + .../headless/api/pojo/QueryConfig.java | 13 + .../headless/api/pojo/QueryStat.java | 13 +- .../api/pojo/TagTypeDefaultConfig.java | 18 + .../headless/api/pojo/TimeDefaultConfig.java | 26 + .../headless/api/pojo/ViewDetail.java | 11 + .../headless/api/pojo/ViewModelConfig.java | 27 + .../headless/api/pojo/enums/SchemaType.java | 7 + .../{ViewInfoReq.java => CanvasReq.java} | 5 +- .../api/pojo/request/MetricQueryReq.java | 1 - .../pojo/request/ModelSchemaFilterReq.java | 14 - .../api/pojo/request/ParseSqlReq.java | 5 +- .../api/pojo/request/QueryMultiStructReq.java | 10 +- .../api/pojo/request/QuerySqlReq.java | 4 +- .../api/pojo/request/QueryStructReq.java | 24 +- .../api/pojo/request/SchemaFilterReq.java | 15 + .../api/pojo/request/SemanticQueryReq.java | 17 +- .../api/pojo/request/ViewFilterReq.java | 33 + .../headless/api/pojo/request/ViewReq.java | 41 + .../headless/api/pojo/response/AppResp.java | 12 + ...emaRelaResp.java => CanvasSchemaResp.java} | 2 +- .../headless/api/pojo/response/ModelResp.java | 4 +- .../api/pojo/response/SemanticSchemaResp.java | 38 + .../headless/api/pojo/response/ViewResp.java | 61 ++ .../api/pojo/response/ViewSchemaResp.java | 35 + .../core/parser/DefaultQueryParser.java | 16 +- .../core/parser/calcite/CalciteSqlParser.java | 7 +- .../parser/calcite/s2sql/SemanticModel.java | 5 +- .../parser/calcite/schema/SchemaBuilder.java | 15 +- .../parser/calcite/schema/SemanticSchema.java | 24 +- .../converter/CalculateAggConverter.java | 10 +- .../converter/ParserDefaultConverter.java | 8 +- .../headless/core/pojo/QueryStatement.java | 9 +- .../headless/core/utils/SqlGenerateUtils.java | 54 +- .../server/aspect/AuthCheckBaseAspect.java | 30 +- .../server/aspect/DimValueAspect.java | 18 +- .../server/aspect/S2DataPermissionAspect.java | 70 +- .../listener/MetaEmbeddingListener.java | 10 +- .../server/manager/ModelYamlManager.java | 6 +- .../server/manager/SemanticSchemaManager.java | 91 +-- .../{ViewInfoDO.java => CanvasDO.java} | 4 +- .../server/persistence/dataobject/ViewDO.java | 46 ++ .../dataobject/ViewInfoDOExample.java | 740 ------------------ ...wInfoDOMapper.java => CanvasDOMapper.java} | 4 +- .../persistence/mapper/ViewDOMapper.java | 11 + .../repository/CanvasRepository.java | 18 + .../repository/ViewInfoRepository.java | 18 - .../repository/impl/CanvasRepositoryImpl.java | 48 ++ .../repository/impl/StatRepositoryImpl.java | 4 +- .../impl/ViewInfoRepositoryImpl.java | 48 -- .../headless/server/pojo/MetaFilter.java | 4 + .../server/rest/CanvasController.java | 57 ++ .../server/rest/SchemaController.java | 11 +- .../headless/server/rest/ViewController.java | 64 ++ .../server/rest/ViewInfoController.java | 57 -- .../headless/server/service/Catalog.java | 32 +- .../headless/server/service/ModelService.java | 18 +- .../server/service/SchemaService.java | 22 +- .../headless/server/service/ViewService.java | 21 + ...erviceImpl.java => CanvasServiceImpl.java} | 60 +- .../server/service/impl/CatalogImpl.java | 87 +- .../service/impl/DimensionServiceImpl.java | 24 +- .../service/impl/DownloadServiceImpl.java | 88 ++- .../service/impl/MetricServiceImpl.java | 47 +- .../server/service/impl/ModelServiceImpl.java | 141 +--- .../server/service/impl/QueryServiceImpl.java | 93 +-- .../service/impl/SchemaServiceImpl.java | 258 +++++- .../server/service/impl/ViewServiceImpl.java | 100 +++ .../server/utils/DimensionConverter.java | 9 + .../server/utils/MetricConverter.java | 9 + .../server/utils/QueryReqConverter.java | 134 ++-- .../server/utils/QueryStructUtils.java | 50 +- .../headless/server/utils/QueryUtils.java | 39 +- .../headless/server/utils/StatUtils.java | 48 +- .../src/main/resources/mapper/StatMapper.xml | 5 +- .../calcite/HeadlessParserServiceTest.java | 15 +- .../service/DownloadServiceImplTest.java | 22 +- .../server/service/MetricServiceImplTest.java | 4 +- .../server/service/ModelServiceImplTest.java | 4 +- .../main/resources/META-INF/spring.factories | 2 +- .../tencent/supersonic/ChatDemoLoader.java | 297 +------ .../supersonic/ModelDemoDataLoader.java | 73 ++ .../main/resources/META-INF/spring.factories | 10 +- .../resources/config.update/sql-update.sql | 23 +- .../data/dictionary/custom/DimValue_1_2.txt | 22 +- .../data/dictionary/custom/DimValue_1_3.txt | 12 +- .../data/dictionary/custom/DimValue_4_9.txt | 18 +- .../src/main/resources/db/data-h2.sql | 2 +- .../src/main/resources/db/data-mysql.sql | 2 +- .../src/main/resources/db/schema-h2.sql | 27 +- .../src/main/resources/db/schema-mysql.sql | 47 +- .../com/tencent/supersonic/chat/BaseTest.java | 11 +- .../supersonic/chat/MetricInterpretTest.java | 27 - .../tencent/supersonic/chat/MetricTest.java | 12 +- .../supersonic/chat/MockConfiguration.java | 6 +- .../com/tencent/supersonic/chat/TagTest.java | 12 +- .../supersonic/chat/mapper/MapperTest.java | 4 +- .../chat/model/MetricServiceImplTest.java | 32 - .../supersonic/headless/ExplainTest.java | 11 +- .../tencent/supersonic/util/DataUtils.java | 19 +- .../test/resources/META-INF/spring.factories | 10 +- .../data/dictionary/custom/DimValue_10_20.txt | 8 + .../data/dictionary/custom/DimValue_10_22.txt | 8 + .../data/dictionary/custom/DimValue_1_2.txt | 22 +- .../data/dictionary/custom/DimValue_1_3.txt | 12 +- .../data/dictionary/custom/DimValue_4_9.txt | 18 +- .../data/dictionary/custom/DimValue_9_15.txt | 5 + .../data/dictionary/custom/DimValue_9_16.txt | 4 + .../data/dictionary/custom/DimValue_9_18.txt | 7 + .../data/dictionary/custom/DimValue_9_19.txt | 7 + .../src/test/resources/db/data-h2.sql | 2 +- .../src/test/resources/db/schema-h2.sql | 26 +- pom.xml | 2 +- 237 files changed, 3205 insertions(+), 4479 deletions(-) delete mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java rename chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/{ModelSchema.java => ViewSchema.java} (66%) rename chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/{ModelInfo.java => ViewInfo.java} (75%) delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/DataAnalyticsTool.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/FromCorrector.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ModelInfoStat.java create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java rename chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/{ModelSchemaBuilder.java => ViewSchemaBuilder.java} (84%) delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelClusterMapper.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicModelResolver.java create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelResolver.java rename chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/{ModelMatchResult.java => ViewMatchResult.java} (82%) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ModelClusterBuilder.java delete mode 100644 chat/server/src/main/resources/mapper/PluginDOMapper.xml delete mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java rename headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/{ViewInfoReq.java => CanvasReq.java} (92%) delete mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelSchemaFilterReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java rename headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/{ModelSchemaRelaResp.java => CanvasSchemaResp.java} (88%) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/{ViewInfoDO.java => CanvasDO.java} (93%) create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDOExample.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/{ViewInfoDOMapper.java => CanvasDOMapper.java} (74%) create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/CanvasRepository.java delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ViewInfoRepository.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/CanvasRepositoryImpl.java delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ViewInfoRepositoryImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewInfoController.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/{ViewInfoServiceImpl.java => CanvasServiceImpl.java} (55%) create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java delete mode 100644 launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java delete mode 100644 launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt create mode 100644 launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java index 1dae45ce6..2fb27fafb 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java @@ -17,7 +17,7 @@ import java.util.List; @NoArgsConstructor public class SchemaElement implements Serializable { - private Long model; + private Long view; private Long id; private String name; private String bizName; @@ -40,7 +40,7 @@ public class SchemaElement implements Serializable { return false; } SchemaElement schemaElement = (SchemaElement) o; - return Objects.equal(model, schemaElement.model) && Objects.equal(id, + return Objects.equal(view, schemaElement.view) && Objects.equal(id, schemaElement.id) && Objects.equal(name, schemaElement.name) && Objects.equal(bizName, schemaElement.bizName) && Objects.equal(type, schemaElement.type); @@ -48,7 +48,7 @@ public class SchemaElement implements Serializable { @Override public int hashCode() { - return Objects.hashCode(model, id, name, bizName, type); + return Objects.hashCode(view, id, name, bizName, type); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java index b5ae5b6b1..ab254c538 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; public enum SchemaElementType { - MODEL, + VIEW, METRIC, DIMENSION, VALUE, diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java index 057599569..f75ef845a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; +import com.google.common.collect.Lists; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -7,25 +9,25 @@ import java.util.Set; public class SchemaMapInfo { - private Map> modelElementMatches = new HashMap<>(); + private Map> viewElementMatches = new HashMap<>(); - public Set getMatchedModels() { - return modelElementMatches.keySet(); + public Set getMatchedViewInfos() { + return viewElementMatches.keySet(); } - public List getMatchedElements(Long model) { - return modelElementMatches.get(model); + public List getMatchedElements(Long view) { + return viewElementMatches.getOrDefault(view, Lists.newArrayList()); } - public Map> getModelElementMatches() { - return modelElementMatches; + public Map> getViewElementMatches() { + return viewElementMatches; } - public void setModelElementMatches(Map> modelElementMatches) { - this.modelElementMatches = modelElementMatches; + public void setViewElementMatches(Map> viewElementMatches) { + this.viewElementMatches = viewElementMatches; } - public void setMatchedElements(Long model, List elementMatches) { - modelElementMatches.put(model, elementMatches); + public void setMatchedElements(Long view, List elementMatches) { + viewElementMatches.put(view, elementMatches); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java deleted file mode 100644 index 43723e40f..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo; - -import com.clickhouse.client.internal.apache.commons.compress.utils.Lists; -import com.tencent.supersonic.common.pojo.ModelCluster; -import lombok.Data; -import org.springframework.util.CollectionUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Data -public class SchemaModelClusterMapInfo { - - private Map> modelElementMatches = new HashMap<>(); - - public Set getMatchedModelClusters() { - return modelElementMatches.keySet(); - } - - public List getMatchedElements(Long modelId) { - for (String key : modelElementMatches.keySet()) { - if (ModelCluster.getModelIdFromKey(key).contains(modelId)) { - return modelElementMatches.get(key); - } - } - return Lists.newArrayList(); - } - - public List getMatchedElements(String modelCluster) { - return modelElementMatches.get(modelCluster); - } - - public Map> getModelElementMatches() { - return modelElementMatches; - } - - public Map> getElementMatchesByModelIds(Set modelIds) { - if (CollectionUtils.isEmpty(modelIds)) { - return modelElementMatches; - } - Map> modelElementMatchesFiltered = new HashMap<>(); - for (String key : modelElementMatches.keySet()) { - for (Long modelId : modelIds) { - if (ModelCluster.getModelIdFromKey(key).contains(modelId)) { - modelElementMatchesFiltered.put(key, modelElementMatches.get(key)); - } - } - } - return modelElementMatchesFiltered; - } - - public void setModelElementMatches(Map> modelElementMatches) { - this.modelElementMatches = modelElementMatches; - } - - public void setMatchedElements(String modelCluster, List elementMatches) { - modelElementMatches.put(modelCluster, elementMatches); - } -} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index e272fe590..f4f8b783d 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -5,7 +5,6 @@ import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; @@ -26,7 +25,7 @@ public class SemanticParseInfo { private Integer id; private String queryMode; - private ModelCluster model = new ModelCluster(); + private SchemaElement view; private Set metrics = new TreeSet<>(new SchemaNameLengthComparator()); private Set dimensions = new LinkedHashSet(); private SchemaElement entity; @@ -44,20 +43,6 @@ public class SemanticParseInfo { private SqlInfo sqlInfo = new SqlInfo(); private QueryType queryType = QueryType.ID; - public String getModelClusterKey() { - if (model == null) { - return ""; - } - return model.getKey(); - } - - public String getModelName() { - if (model == null) { - return ""; - } - return model.getName(); - } - private static class SchemaNameLengthComparator implements Comparator { @Override @@ -86,27 +71,11 @@ public class SemanticParseInfo { return metrics; } - private Map getModelElementCountMap() { - Map elementCountMap = new HashMap<>(); - elementMatches.stream().filter(element -> element.getElement().getModel() != null) - .forEach(element -> { - int count = elementCountMap.getOrDefault(element.getElement().getModel(), 0); - elementCountMap.put(element.getElement().getModel(), count + 1); - }); - return elementCountMap; - } - - public Long getModelId() { - Map elementCountMap = getModelElementCountMap(); - Long modelId = -1L; - int maxCnt = 0; - for (Long model : elementCountMap.keySet()) { - if (elementCountMap.get(model) > maxCnt) { - maxCnt = elementCountMap.get(model); - modelId = model; - } + public Long getViewId() { + if (view == null) { + return null; } - return modelId; + return view.getView(); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 3e0301e5a..db103110f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; +import org.springframework.util.CollectionUtils; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -7,20 +9,18 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; public class SemanticSchema implements Serializable { - private List modelSchemaList; + private List viewSchemaList; - public SemanticSchema(List modelSchemaList) { - this.modelSchemaList = modelSchemaList; + public SemanticSchema(List viewSchemaList) { + this.viewSchemaList = viewSchemaList; } - public void add(ModelSchema schema) { - modelSchemaList.add(schema); + public void add(ViewSchema schema) { + viewSchemaList.add(schema); } public SchemaElement getElement(SchemaElementType elementType, long elementID) { @@ -30,8 +30,8 @@ public class SemanticSchema implements Serializable { case ENTITY: element = getElementsById(elementID, getEntities()); break; - case MODEL: - element = getElementsById(elementID, getModels()); + case VIEW: + element = getElementsById(elementID, getViews()); break; case METRIC: element = getElementsById(elementID, getMetrics()); @@ -59,8 +59,8 @@ public class SemanticSchema implements Serializable { case ENTITY: element = getElementsByNameOrAlias(name, getEntities()); break; - case MODEL: - element = getElementsByNameOrAlias(name, getModels()); + case VIEW: + element = getElementsByNameOrAlias(name, getViews()); break; case METRIC: element = getElementsByNameOrAlias(name, getMetrics()); @@ -81,29 +81,29 @@ public class SemanticSchema implements Serializable { } } - public Map getModelIdToName() { - return modelSchemaList.stream() - .collect(Collectors.toMap(a -> a.getModel().getId(), a -> a.getModel().getName(), (k1, k2) -> k1)); + public Map getViewIdToName() { + return viewSchemaList.stream() + .collect(Collectors.toMap(a -> a.getView().getId(), a -> a.getView().getName(), (k1, k2) -> k1)); } public List getDimensionValues() { List dimensionValues = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues())); + viewSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues())); return dimensionValues; } public List getDimensions() { List dimensions = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions())); + viewSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions())); return dimensions; } - public List getDimensions(Set modelIds) { + public List getDimensions(Long viewId) { List dimensions = getDimensions(); - return getElementsByModelId(modelIds, dimensions); + return getElementsByViewId(viewId, dimensions); } - public SchemaElement getDimensions(Long id) { + public SchemaElement getDimension(Long id) { List dimensions = getDimensions(); Optional dimension = getElementsById(id, dimensions); return dimension.orElse(null); @@ -111,43 +111,43 @@ public class SemanticSchema implements Serializable { public List getTags() { List tags = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); + viewSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); return tags; } - public List getTags(Set modelIds) { + public List getTags(Long viewId) { List tags = new ArrayList<>(); - modelSchemaList.stream().filter(schemaElement -> - modelIds.contains(schemaElement.getModel().getModel())) + viewSchemaList.stream().filter(schemaElement -> + viewId.equals(schemaElement.getView().getView())) .forEach(d -> tags.addAll(d.getTags())); return tags; } public List getMetrics() { List metrics = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics())); + viewSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics())); return metrics; } - public List getMetrics(Set modelIds) { + public List getMetrics(Long viewId) { List metrics = getMetrics(); - return getElementsByModelId(modelIds, metrics); + return getElementsByViewId(viewId, metrics); } public List getEntities() { List entities = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> entities.add(d.getEntity())); + viewSchemaList.stream().forEach(d -> entities.add(d.getEntity())); return entities; } - public List getEntities(Set modelIds) { + public List getEntities(Long viewId) { List entities = getEntities(); - return getElementsByModelId(modelIds, entities); + return getElementsByViewId(viewId, entities); } - private List getElementsByModelId(Set modelIds, List elements) { + private List getElementsByViewId(Long viewId, List elements) { return elements.stream() - .filter(schemaElement -> modelIds.contains(schemaElement.getModel())) + .filter(schemaElement -> viewId.equals(schemaElement.getView())) .collect(Collectors.toList()); } @@ -165,25 +165,30 @@ public class SemanticSchema implements Serializable { ).findFirst(); } - public List getModels() { - List models = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> models.add(d.getModel())); - return models; + public SchemaElement getView(Long viewId) { + List views = getViews(); + return getElementsById(viewId, views).orElse(null); } - public Map getBizNameToName(Set modelIds) { + public List getViews() { + List views = new ArrayList<>(); + viewSchemaList.stream().forEach(d -> views.add(d.getView())); + return views; + } + + public Map getBizNameToName(Long viewId) { List allElements = new ArrayList<>(); - allElements.addAll(getDimensions(modelIds)); - allElements.addAll(getMetrics(modelIds)); + allElements.addAll(getDimensions(viewId)); + allElements.addAll(getMetrics(viewId)); return allElements.stream() .collect(Collectors.toMap(SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1)); } - public Map getModelSchemaMap() { - if (CollectionUtils.isEmpty(modelSchemaList)) { + public Map getViewSchemaMap() { + if (CollectionUtils.isEmpty(viewSchemaList)) { return new HashMap<>(); } - return modelSchemaList.stream().collect(Collectors.toMap(modelSchema - -> modelSchema.getModel().getModel(), modelSchema -> modelSchema)); + return viewSchemaList.stream().collect(Collectors.toMap(viewSchema + -> viewSchema.getView().getView(), viewSchema -> viewSchema)); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java similarity index 66% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java index 4719a7cdf..9c6c88a04 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java @@ -1,26 +1,24 @@ package com.tencent.supersonic.chat.api.pojo; -import com.google.common.collect.Sets; -import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import lombok.Data; -import org.springframework.util.CollectionUtils; -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; @Data -public class ModelSchema { +public class ViewSchema { - private SchemaElement model; + private SchemaElement view; private Set metrics = new HashSet<>(); private Set dimensions = new HashSet<>(); private Set dimensionValues = new HashSet<>(); private Set tags = new HashSet<>(); private SchemaElement entity = new SchemaElement(); - private List modelRelas = new ArrayList<>(); + private QueryConfig queryConfig; public SchemaElement getElement(SchemaElementType elementType, long elementID) { Optional element = Optional.empty(); @@ -29,8 +27,8 @@ public class ModelSchema { case ENTITY: element = Optional.ofNullable(entity); break; - case MODEL: - element = Optional.of(model); + case VIEW: + element = Optional.of(view); break; case METRIC: element = metrics.stream().filter(e -> e.getId() == elementID).findFirst(); @@ -61,8 +59,8 @@ public class ModelSchema { case ENTITY: element = Optional.ofNullable(entity); break; - case MODEL: - element = Optional.of(model); + case VIEW: + element = Optional.of(view); break; case METRIC: element = metrics.stream().filter(e -> name.equals(e.getName())).findFirst(); @@ -83,16 +81,31 @@ public class ModelSchema { } } - public Set getModelClusterSet() { - if (CollectionUtils.isEmpty(modelRelas)) { - return Sets.newHashSet(); + public TimeDefaultConfig getTagTypeTimeDefaultConfig() { + if (queryConfig == null) { + return null; } - Set modelClusterSet = new HashSet<>(); - modelRelas.forEach(modelRela -> { - modelClusterSet.add(modelRela.getToModelId()); - modelClusterSet.add(modelRela.getFromModelId()); - }); - return modelClusterSet; + if (queryConfig.getTagTypeDefaultConfig() == null) { + return null; + } + return queryConfig.getTagTypeDefaultConfig().getTimeDefaultConfig(); + } + + public TimeDefaultConfig getMetricTypeTimeDefaultConfig() { + if (queryConfig == null) { + return null; + } + if (queryConfig.getMetricTypeDefaultConfig() == null) { + return null; + } + return queryConfig.getMetricTypeDefaultConfig().getTimeDefaultConfig(); + } + + public TagTypeDefaultConfig getTagTypeDefaultConfig() { + if (queryConfig == null) { + return null; + } + return queryConfig.getTagTypeDefaultConfig(); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java index 33ca4889c..a5d9925bd 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.request; -import com.tencent.supersonic.common.pojo.Constants; import lombok.Data; import java.util.ArrayList; @@ -13,26 +12,5 @@ public class ChatDefaultConfigReq { private List dimensionIds = new ArrayList<>(); private List metricIds = new ArrayList<>(); - /** - * default time span unit - */ - private Integer unit = 1; - - /** - * default time type: day - * DAY, WEEK, MONTH, YEAR - */ - private String period = Constants.DAY; - - private TimeMode timeMode = TimeMode.LAST; - - public enum TimeMode { - /** - * date mode - * LAST - a certain time - * RECENT - a period time - */ - LAST, RECENT - } } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java index 72b31244a..48457a279 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java @@ -13,7 +13,7 @@ public class PluginQueryReq { private String type; - private String model; + private String view; private String pattern; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java index 3ce7e3d1f..7c5c6c9f2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SimilarQueryReq.java @@ -18,7 +18,7 @@ public class SimilarQueryReq { private String queryText; - private String modelId; + private Long viewId; private Integer agentId; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java index 5aa2b2d32..82dc7c5d3 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq.TimeMode; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeMode; import lombok.Data; import java.util.List; @@ -21,7 +21,7 @@ public class ChatDefaultRichConfigResp { private Integer unit = 1; /** - * default time type: day + * default time type: * DAY, WEEK, MONTH, YEAR */ private String period = Constants.DAY; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java index 7b22daf73..04576e892 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java @@ -1,8 +1,12 @@ package com.tencent.supersonic.chat.api.pojo.response; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor public class DataInfo { private Integer itemId; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java index f7e606469..dd6c07195 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java @@ -1,14 +1,16 @@ package com.tencent.supersonic.chat.api.pojo.response; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import lombok.Data; @Data public class EntityInfo { - private ModelInfo modelInfo = new ModelInfo(); + private ViewInfo viewInfo = new ViewInfo(); private List dimensions = new ArrayList<>(); private List metrics = new ArrayList<>(); private String entityId; + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java similarity index 75% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java index 5944e7f5f..d772b3ced 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ViewInfo.java @@ -6,7 +6,7 @@ import java.io.Serializable; import java.util.List; @Data -public class ModelInfo extends DataInfo implements Serializable { +public class ViewInfo extends DataInfo implements Serializable { private List words; private String primaryKey; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java index 7f74b798e..a1c24f55f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/Agent.java @@ -4,6 +4,9 @@ package com.tencent.supersonic.chat.core.agent; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; +import lombok.Data; +import org.springframework.util.CollectionUtils; + import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -11,8 +14,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.Data; -import org.springframework.util.CollectionUtils; @Data public class Agent extends RecordInfo { @@ -51,8 +52,8 @@ public class Agent extends RecordInfo { return enableSearch != null && enableSearch == 1; } - public static boolean containsAllModel(Set detectModelIds) { - return !CollectionUtils.isEmpty(detectModelIds) && detectModelIds.contains(-1L); + public static boolean containsAllModel(Set detectViewIds) { + return !CollectionUtils.isEmpty(detectViewIds) && detectViewIds.contains(-1L); } public List getParserTools(AgentToolType agentToolType) { @@ -64,12 +65,12 @@ public class Agent extends RecordInfo { .collect(Collectors.toList()); } - public Set getModelIds(AgentToolType agentToolType) { + public Set getViewIds(AgentToolType agentToolType) { List commonAgentTools = getParserTools(agentToolType); if (CollectionUtils.isEmpty(commonAgentTools)) { return new HashSet<>(); } - return commonAgentTools.stream().map(NL2SQLTool::getModelIds) + return commonAgentTools.stream().map(NL2SQLTool::getViewIds) .filter(modelIds -> !CollectionUtils.isEmpty(modelIds)) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/DataAnalyticsTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/DataAnalyticsTool.java deleted file mode 100644 index 559dd719a..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/DataAnalyticsTool.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.chat.core.agent; - -import lombok.Data; - - -@Data -public class DataAnalyticsTool extends AgentTool { - - private Long modelId; - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java index bff9c120d..eff5f11d0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/NL2SQLTool.java @@ -1,16 +1,17 @@ package com.tencent.supersonic.chat.core.agent; -import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @NoArgsConstructor @AllArgsConstructor public class NL2SQLTool extends AgentTool { - protected List modelIds; + protected List viewIds; } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java index bb8374e51..45c6a7323 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/agent/RuleParserTool.java @@ -15,7 +15,7 @@ public class RuleParserTool extends NL2SQLTool { private List queryTypes; public boolean isContainsAllModel() { - return CollectionUtils.isNotEmpty(modelIds) && modelIds.contains(-1L); + return CollectionUtils.isNotEmpty(viewIds) && viewIds.contains(-1L); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java index 8566b0851..2215efb58 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java @@ -1,14 +1,19 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -16,10 +21,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.util.CollectionUtils; /** * basic semantic correction functionality, offering common methods and an @@ -42,7 +43,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { public abstract void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo); - protected Map getFieldNameMap(QueryContext queryContext, Set modelIds) { + protected Map getFieldNameMap(QueryContext queryContext, Long viewId) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); @@ -52,7 +53,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { // support fieldName and field alias Map result = dbAllFields.stream() - .filter(entry -> modelIds.contains(entry.getModel())) + .filter(entry -> viewId.equals(entry.getView())) .flatMap(schemaElement -> { Set elements = new HashSet<>(); elements.add(schemaElement.getName()); @@ -100,9 +101,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { protected void addAggregateToMetric(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { //add aggregate to all metric String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - Set modelIds = semanticParseInfo.getModel().getModelIds(); - - List metrics = getMetricElements(queryContext, modelIds); + Long viewId = semanticParseInfo.getView().getView(); + List metrics = getMetricElements(queryContext, viewId); Map metricToAggregate = metrics.stream() .map(schemaElement -> { @@ -127,9 +127,9 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql); } - protected List getMetricElements(QueryContext queryContext, Set modelIds) { + protected List getMetricElements(QueryContext queryContext, Long viewId) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - return semanticSchema.getMetrics(modelIds); + return semanticSchema.getMetrics(viewId); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/FromCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/FromCorrector.java deleted file mode 100644 index bc71e09ab..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/FromCorrector.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tencent.supersonic.chat.core.corrector; - -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import lombok.extern.slf4j.Slf4j; -/** - * Perform SQL corrections on the "From" section in S2SQL. - */ -@Slf4j -public class FromCorrector extends BaseSemanticCorrector { - - @Override - public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - String modelName = semanticParseInfo.getModel().getName(); - String correctSql = SqlParserReplaceHelper - .replaceTable(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), modelName); - semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctSql); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java index ca20d2cff..7fd95e1ef 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java @@ -1,18 +1,19 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /** * Perform SQL corrections on the "Group by" section in S2SQL. @@ -28,8 +29,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { } private void addGroupByFields(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Set modelIds = semanticParseInfo.getModel().getModelIds(); - + Long viewId = semanticParseInfo.getViewId(); //add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectS2SQL(); @@ -41,7 +41,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { return; } //add alias field name - Set dimensions = semanticSchema.getDimensions(modelIds).stream() + Set dimensions = semanticSchema.getDimensions(viewId).stream() .flatMap( schemaElement -> { Set elements = new HashSet<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java index 22a8250d6..89e57f0ca 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java @@ -1,18 +1,19 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + /** * Perform SQL corrections on the "Having" section in S2SQL. */ @@ -31,11 +32,11 @@ public class HavingCorrector extends BaseSemanticCorrector { } private void addHaving(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Set modelIds = semanticParseInfo.getModel().getModelIds(); + Long viewId = semanticParseInfo.getView().getView(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - Set metrics = semanticSchema.getMetrics(modelIds).stream() + Set metrics = semanticSchema.getMetrics(viewId).stream() .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); if (CollectionUtils.isEmpty(metrics)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java index 04fc8d3f1..85f3ef730 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java @@ -1,21 +1,22 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /** * Perform schema corrections on the Schema information in S2QL. @@ -51,7 +52,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { } private void correctFieldName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - Map fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getModel().getModelIds()); + Map fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getViewId()); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap); sqlInfo.setCorrectS2SQL(sql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java index 3206e7d27..41604054d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java @@ -1,24 +1,18 @@ package com.tencent.supersonic.chat.core.corrector; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.core.parser.sql.llm.S2SqlDateHelper; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -27,6 +21,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + /** * Perform SQL corrections on the "Where" section in S2SQL. */ @@ -73,7 +73,7 @@ public class WhereCorrector extends BaseSemanticCorrector { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); List whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL); if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { - String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getModelId()); + String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getViewId()); if (StringUtils.isNotBlank(currentDate)) { correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL); correctS2SQL = SqlParserAddHelper.addWhere( @@ -99,8 +99,8 @@ public class WhereCorrector extends BaseSemanticCorrector { private void updateFieldValueByTechName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - Set modelIds = semanticParseInfo.getModel().getModelIds(); - List dimensions = semanticSchema.getDimensions(modelIds); + Long viewId = semanticParseInfo.getViewId(); + List dimensions = semanticSchema.getDimensions(viewId); if (CollectionUtils.isEmpty(dimensions)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java index effd8a42e..be6bb2b1c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/LoadRemoveService.java @@ -1,16 +1,17 @@ package com.tencent.supersonic.chat.core.knowledge; import com.tencent.supersonic.chat.core.utils.NatureHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Set; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + @Data @Service public class LoadRemoveService { @@ -31,7 +32,7 @@ public class LoadRemoveService { if (Objects.isNull(nature)) { return false; } - Long modelId = NatureHelper.getModelId(nature); + Long modelId = NatureHelper.getViewId(nature); if (Objects.nonNull(modelId)) { return !detectModelIds.contains(modelId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ModelInfoStat.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ModelInfoStat.java deleted file mode 100644 index 57feeaf51..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ModelInfoStat.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tencent.supersonic.chat.core.knowledge; - -import java.io.Serializable; -import lombok.Builder; -import lombok.Data; -import lombok.ToString; - -@Data -@ToString -@Builder -public class ModelInfoStat implements Serializable { - - private long modelCount; - - private long metricModelCount; - - private long dimensionModelCount; - - private long dimensionValueModelCount; - -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java new file mode 100644 index 000000000..d420e1995 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/ViewInfoStat.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.chat.core.knowledge; + +import lombok.Builder; +import lombok.Data; +import lombok.ToString; + +import java.io.Serializable; + +@Data +@ToString +@Builder +public class ViewInfoStat implements Serializable { + + private long viewCount; + + private long metricViewCount; + + private long dimensionViewCount; + + private long dimensionValueViewCount; + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java index 9d1bd08bf..e000aef00 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/DimensionWordBuilder.java @@ -4,13 +4,14 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.ArrayList; -import java.util.List; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; + /** * dimension word nature */ @@ -37,11 +38,11 @@ public class DimensionWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long domainId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long viewId = schemaElement.getView(); + String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.DIMENSION.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + domainId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java index 0d5c14e12..1f14d7971 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/EntityWordBuilder.java @@ -5,12 +5,13 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; + /** * dimension value wordNature */ @@ -26,8 +27,8 @@ public class EntityWordBuilder extends BaseWordBuilder { return result; } - Long domain = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + domain + DictWordType.NATURE_SPILT + schemaElement.getId() + Long view = schemaElement.getView(); + String nature = DictWordType.NATURE_SPILT + view + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType(); if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java index 591e20a52..6fe0cecd6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/MetricWordBuilder.java @@ -4,13 +4,14 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.ArrayList; -import java.util.List; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; + /** * Metric DictWord */ @@ -37,11 +38,11 @@ public class MetricWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long modelId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long viewId = schemaElement.getView(); + String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.METRIC.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java index 1df55d78e..d388dd3a3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ModelWordBuilder.java @@ -4,11 +4,12 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; +import java.util.List; + /** * model word nature */ @@ -20,13 +21,13 @@ public class ModelWordBuilder extends BaseWordBuilder { public List doGet(String word, SchemaElement schemaElement) { List result = Lists.newArrayList(); //modelName - DictWord dictWord = buildDictWord(word, schemaElement.getModel()); + DictWord dictWord = buildDictWord(word, schemaElement.getView()); result.add(dictWord); //alias List aliasList = schemaElement.getAlias(); if (CollectionUtils.isNotEmpty(aliasList)) { for (String alias : aliasList) { - result.add(buildDictWord(alias, schemaElement.getModel())); + result.add(buildDictWord(alias, schemaElement.getView())); } } return result; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java index 82ba3a0a2..95f5e8aea 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/ValueWordBuilder.java @@ -5,12 +5,13 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; + /** * dimension value wordNature */ @@ -26,8 +27,8 @@ public class ValueWordBuilder extends BaseWordBuilder { schemaElement.getAlias().stream().forEach(value -> { DictWord dictWord = new DictWord(); - Long modelId = schemaElement.getModel(); - String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId(); + Long viewId = schemaElement.getView(); + String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId(); dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); dictWord.setWord(value); result.add(dictWord); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java index 273b5f29c..f13e2417c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/builder/WordBuilderFactory.java @@ -15,7 +15,7 @@ public class WordBuilderFactory { static { wordNatures.put(DictWordType.DIMENSION, new DimensionWordBuilder()); wordNatures.put(DictWordType.METRIC, new MetricWordBuilder()); - wordNatures.put(DictWordType.MODEL, new ModelWordBuilder()); + wordNatures.put(DictWordType.VIEW, new ModelWordBuilder()); wordNatures.put(DictWordType.ENTITY, new EntityWordBuilder()); wordNatures.put(DictWordType.VALUE, new ValueWordBuilder()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java index 2b8c717dc..33d0d3010 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/BaseSemanticInterpreter.java @@ -2,66 +2,67 @@ package com.tencent.supersonic.chat.core.knowledge.semantic; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + @Slf4j public abstract class BaseSemanticInterpreter implements SemanticInterpreter { - protected final Cache> modelSchemaCache = + protected final Cache> viewSchemaCache = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); @SneakyThrows - public List fetchModelSchema(List ids, Boolean cacheEnable) { + public List fetchViewSchema(List ids, Boolean cacheEnable) { if (cacheEnable) { - return modelSchemaCache.get(String.valueOf(ids), () -> { - List data = doFetchModelSchema(ids); - modelSchemaCache.put(String.valueOf(ids), data); + return viewSchemaCache.get(String.valueOf(ids), () -> { + List data = doFetchViewSchema(ids); + viewSchemaCache.put(String.valueOf(ids), data); return data; }); } - List data = doFetchModelSchema(ids); - return data; + return doFetchViewSchema(ids); } @Override - public ModelSchema getModelSchema(Long model, Boolean cacheEnable) { + public ViewSchema getViewSchema(Long viewId, Boolean cacheEnable) { List ids = new ArrayList<>(); - ids.add(model); - List modelSchemaResps = fetchModelSchema(ids, cacheEnable); - if (!CollectionUtils.isEmpty(modelSchemaResps)) { - Optional modelSchemaResp = modelSchemaResps.stream() - .filter(d -> d.getId().equals(model)).findFirst(); - if (modelSchemaResp.isPresent()) { - ModelSchemaResp modelSchema = modelSchemaResp.get(); - return ModelSchemaBuilder.build(modelSchema); + ids.add(viewId); + List viewSchemaResps = fetchViewSchema(ids, cacheEnable); + if (!CollectionUtils.isEmpty(viewSchemaResps)) { + Optional viewSchemaResp = viewSchemaResps.stream() + .filter(d -> d.getId().equals(viewId)).findFirst(); + if (viewSchemaResp.isPresent()) { + ViewSchemaResp viewSchema = viewSchemaResp.get(); + return ViewSchemaBuilder.build(viewSchema); } } return null; } @Override - public List getModelSchema() { - return getModelSchema(new ArrayList<>()); + public List getViewSchema() { + return getViewSchema(new ArrayList<>()); } @Override - public List getModelSchema(List ids) { - List domainSchemaList = new ArrayList<>(); + public List getViewSchema(List ids) { + List domainSchemaList = new ArrayList<>(); - for (ModelSchemaResp resp : fetchModelSchema(ids, true)) { - domainSchemaList.add(ModelSchemaBuilder.build(resp)); + for (ViewSchemaResp resp : fetchViewSchema(ids, true)) { + domainSchemaList.add(ViewSchemaBuilder.build(resp)); } return domainSchemaList; } - protected abstract List doFetchModelSchema(List ids); + protected abstract List doFetchViewSchema(List ids); + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java index b81669f8f..d6989d091 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java @@ -2,34 +2,33 @@ package com.tencent.supersonic.chat.core.knowledge.semantic; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.service.SchemaService; -import java.util.ArrayList; -import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.List; + @Slf4j public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @@ -44,7 +43,7 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { QuerySqlReq querySqlReq = new QuerySqlReq(); querySqlReq.setSql(queryStructReq.getCorrectS2SQL()); - querySqlReq.setModelIds(queryStructReq.getModelIdSet()); + querySqlReq.setViewId(queryStructReq.getViewId()); querySqlReq.setParams(new ArrayList<>()); return queryByS2SQL(querySqlReq, user); } @@ -68,19 +67,11 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { } @Override - @SneakyThrows - public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { - queryService = ContextUtils.getBean(QueryService.class); - return queryService.queryDimValue(queryDimValueReq, user); - } - - @Override - public List doFetchModelSchema(List ids) { - ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(ids); + public List doFetchViewSchema(List ids) { + ViewFilterReq filter = new ViewFilterReq(); + filter.setViewIds(ids); schemaService = ContextUtils.getBean(SchemaService.class); - User user = User.getFakeUser(); - return schemaService.fetchModelSchema(filter, user); + return schemaService.fetchViewSchema(filter); } @Override @@ -90,9 +81,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public List getModelList(AuthType authType, Long domainId, User user) { + public List getViewList(Long domainId) { schemaService = ContextUtils.getBean(SchemaService.class); - return schemaService.getModelList(user, authType, domainId); + return schemaService.getViewList(domainId); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java index 916a6561f..79f251cd5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java @@ -1,19 +1,11 @@ package com.tencent.supersonic.chat.core.knowledge.semantic; -import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER; - -import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; -import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.core.config.DefaultSemanticConfig; import com.tencent.supersonic.common.pojo.ResultData; -import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.ReturnCode; import com.tencent.supersonic.common.pojo.exception.CommonException; import com.tencent.supersonic.common.util.ContextUtils; @@ -21,10 +13,8 @@ import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.ThreadContext; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; @@ -32,15 +22,9 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -54,6 +38,17 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Objects; + +import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; + @Slf4j public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { @@ -130,50 +125,6 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { throw new CommonException(responseBody.getCode(), responseBody.getMsg()); } - @Override - public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() - + defaultSemanticConfig.getQueryDimValuePath(), - new Gson().toJson(queryDimValueReq)); - } - - @Override - public List doFetchModelSchema(List ids) { - HttpHeaders headers = new HttpHeaders(); - headers.set(UserConstants.INTERNAL, TRUE_LOWER); - headers.setContentType(MediaType.APPLICATION_JSON); - fillToken(headers); - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - - String semanticUrl = defaultSemanticConfig.getSemanticUrl(); - String fetchModelSchemaPath = defaultSemanticConfig.getFetchModelSchemaPath(); - URI requestUrl = UriComponentsBuilder.fromHttpUrl(semanticUrl + fetchModelSchemaPath) - .build().encode().toUri(); - ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(ids); - ParameterizedTypeReference>> responseTypeRef = - new ParameterizedTypeReference>>() { - }; - - HttpEntity entity = new HttpEntity<>(JSON.toJSONString(filter), headers); - - try { - RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - ResponseEntity>> responseEntity = restTemplate.exchange( - requestUrl, HttpMethod.POST, entity, responseTypeRef); - ResultData> responseBody = responseEntity.getBody(); - log.debug("ApiResponse responseBody:{}", responseBody); - if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) { - List data = responseBody.getData(); - return data; - } - } catch (Exception e) { - throw new RuntimeException("fetchModelSchema interface error", e); - } - throw new RuntimeException("fetchModelSchema interface error"); - } - @Override public List getDomainList(User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); @@ -183,19 +134,6 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { return JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class); } - @Override - public List getModelList(AuthType authType, Long domainId, User user) { - if (domainId == null) { - domainId = 0L; - } - DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - String url = String.format("%s?domainId=%s&authType=%s", - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelListPath(), - domainId, authType.toString()); - Object domainDescListObject = fetchHttpResult(url, null, HttpMethod.GET); - return JsonUtil.toList(JsonUtil.toString(domainDescListObject), ModelResp.class); - } - @Override public ExplainResp explain(ExplainSqlReq explainResp, User user) throws Exception { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); @@ -310,4 +248,13 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { return pageInfo; } + @Override + protected List doFetchViewSchema(List ids) { + return null; + } + + @Override + public List getViewList(Long domainId) { + return null; + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java index 4d4c0b407..93fe476cd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java @@ -2,22 +2,20 @@ package com.tencent.supersonic.chat.core.knowledge.semantic; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; +import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import java.util.List; @@ -38,15 +36,13 @@ public interface SemanticInterpreter { SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); - SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user); + SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); - SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); + List getViewSchema(); - List getModelSchema(); + List getViewSchema(List ids); - List getModelSchema(List ids); - - ModelSchema getModelSchema(Long model, Boolean cacheEnable); + ViewSchema getViewSchema(Long model, Boolean cacheEnable); PageInfo getDimensionPage(PageDimensionReq pageDimensionReq); @@ -54,10 +50,10 @@ public interface SemanticInterpreter { List getDomainList(User user); - List getModelList(AuthType authType, Long domainId, User user); - ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; - List fetchModelSchema(List ids, Boolean cacheEnable); + List fetchViewSchema(List ids, Boolean cacheEnable); + + List getViewList(Long domainId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ModelSchemaBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java similarity index 84% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ModelSchemaBuilder.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java index 725ca8315..289b1863b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ModelSchemaBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/ViewSchemaBuilder.java @@ -1,20 +1,21 @@ package com.tencent.supersonic.chat.core.knowledge.semantic; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -22,20 +23,19 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class ModelSchemaBuilder { +public class ViewSchemaBuilder { - public static ModelSchema build(ModelSchemaResp resp) { - ModelSchema modelSchema = new ModelSchema(); + public static ViewSchema build(ViewSchemaResp resp) { + ViewSchema viewSchema = new ViewSchema(); + viewSchema.setQueryConfig(resp.getQueryConfig()); SchemaElement model = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(resp.getId()) .name(resp.getName()) .bizName(resp.getBizName()) - .type(SchemaElementType.MODEL) - .alias(SchemaItem.getAliasList(resp.getAlias())) + .type(SchemaElementType.VIEW) .build(); - modelSchema.setModel(model); - modelSchema.setModelRelas(resp.getModelRelas()); + viewSchema.setView(model); Set metrics = new HashSet<>(); for (MetricSchemaResp metric : resp.getMetrics()) { @@ -43,7 +43,7 @@ public class ModelSchemaBuilder { List alias = SchemaItem.getAliasList(metric.getAlias()); SchemaElement metricToAdd = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(metric.getId()) .name(metric.getName()) .bizName(metric.getBizName()) @@ -56,7 +56,7 @@ public class ModelSchemaBuilder { metrics.add(metricToAdd); } - modelSchema.getMetrics().addAll(metrics); + viewSchema.getMetrics().addAll(metrics); Set dimensions = new HashSet<>(); Set dimensionValues = new HashSet<>(); @@ -83,7 +83,7 @@ public class ModelSchemaBuilder { } SchemaElement dimToAdd = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -95,7 +95,7 @@ public class ModelSchemaBuilder { dimensions.add(dimToAdd); SchemaElement dimValueToAdd = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -106,7 +106,7 @@ public class ModelSchemaBuilder { dimensionValues.add(dimValueToAdd); if (dim.getIsTag() == 1) { SchemaElement tagToAdd = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -118,14 +118,14 @@ public class ModelSchemaBuilder { tags.add(tagToAdd); } } - modelSchema.getDimensions().addAll(dimensions); - modelSchema.getDimensionValues().addAll(dimensionValues); - modelSchema.getTags().addAll(tags); + viewSchema.getDimensions().addAll(dimensions); + viewSchema.getDimensionValues().addAll(dimensionValues); + viewSchema.getTags().addAll(tags); DimSchemaResp dim = resp.getPrimaryKey(); if (dim != null) { SchemaElement entity = SchemaElement.builder() - .model(resp.getId()) + .view(resp.getId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -133,9 +133,9 @@ public class ModelSchemaBuilder { .useCnt(dim.getUseCnt()) .alias(dim.getEntityAlias()) .build(); - modelSchema.setEntity(entity); + viewSchema.setEntity(entity); } - return modelSchema; + return viewSchema; } private static List getRelateSchemaElement(MetricSchemaResp metricSchemaResp) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java index 2da576e78..dc0f98298 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java @@ -1,21 +1,22 @@ package com.tencent.supersonic.chat.core.mapper; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; @Slf4j public abstract class BaseMapper implements SchemaMapper { @@ -25,7 +26,7 @@ public abstract class BaseMapper implements SchemaMapper { String simpleName = this.getClass().getSimpleName(); long startTime = System.currentTimeMillis(); - log.debug("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getModelElementMatches()); + log.debug("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getViewElementMatches()); try { doMap(queryContext); @@ -34,13 +35,13 @@ public abstract class BaseMapper implements SchemaMapper { } long cost = System.currentTimeMillis() - startTime; - log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getModelElementMatches()); + log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getViewElementMatches()); } public abstract void doMap(QueryContext queryContext); public void addToSchemaMap(SchemaMapInfo schemaMap, Long modelId, SchemaElementMatch newElementMatch) { - Map> modelElementMatches = schemaMap.getModelElementMatches(); + Map> modelElementMatches = schemaMap.getViewElementMatches(); List schemaElementMatches = modelElementMatches.putIfAbsent(modelId, new ArrayList<>()); if (schemaElementMatches == null) { schemaElementMatches = modelElementMatches.get(modelId); @@ -66,14 +67,14 @@ public abstract class BaseMapper implements SchemaMapper { } } - public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID, + public SchemaElement getSchemaElement(Long viewId, SchemaElementType elementType, Long elementID, SemanticSchema semanticSchema) { SchemaElement element = new SchemaElement(); - ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId); - if (Objects.isNull(modelSchema)) { + ViewSchema viewSchema = semanticSchema.getViewSchemaMap().get(viewId); + if (Objects.isNull(viewSchema)) { return null; } - SchemaElement elementDb = modelSchema.getElement(elementType, elementID); + SchemaElement elementDb = viewSchema.getElement(elementType, elementID); if (Objects.isNull(elementDb)) { log.info("element is null, elementType:{},elementID:{}", elementType, elementID); return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java index b51ea12aa..887e33deb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java @@ -3,6 +3,12 @@ package com.tencent.supersonic.chat.core.mapper; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.utils.NatureHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -13,11 +19,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -27,15 +28,15 @@ public abstract class BaseMatchStrategy implements MatchStrategy { private MapperHelper mapperHelper; @Override - public Map> match(QueryContext queryContext, List terms, Set detectModelIds) { + public Map> match(QueryContext queryContext, List terms, Set detectViewIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; } - log.debug("terms:{},,detectModelIds:{}", terms, detectModelIds); + log.debug("terms:{},,detectViewIds:{}", terms, detectViewIds); - List detects = detect(queryContext, terms, detectModelIds); + List detects = detect(queryContext, terms, detectViewIds); Map> result = new HashMap<>(); result.put(MatchText.builder().regText(text).detectSegment(text).build(), detects); @@ -102,9 +103,9 @@ public abstract class BaseMatchStrategy implements MatchStrategy { } public List getMatches(QueryContext queryContext, List terms) { - Set detectModelIds = mapperHelper.getModelIds(queryContext.getModelId(), queryContext.getAgent()); - terms = filterByModelIds(terms, detectModelIds); - Map> matchResult = match(queryContext, terms, detectModelIds); + Set viewIds = mapperHelper.getViewIds(queryContext.getViewId(), queryContext.getAgent()); + terms = filterByViewId(terms, viewIds); + Map> matchResult = match(queryContext, terms, viewIds); List matches = new ArrayList<>(); if (Objects.isNull(matchResult)) { return matches; @@ -119,17 +120,17 @@ public abstract class BaseMatchStrategy implements MatchStrategy { return matches; } - public List filterByModelIds(List terms, Set detectModelIds) { + public List filterByViewId(List terms, Set viewIds) { logTerms(terms); - if (CollectionUtils.isNotEmpty(detectModelIds)) { + if (CollectionUtils.isNotEmpty(viewIds)) { terms = terms.stream().filter(term -> { - Long modelId = NatureHelper.getModelId(term.getNature().toString()); - if (Objects.nonNull(modelId)) { - return detectModelIds.contains(modelId); + Long viewId = NatureHelper.getViewId(term.getNature().toString()); + if (Objects.nonNull(viewId)) { + return viewIds.contains(viewId); } return false; }).collect(Collectors.toList()); - log.info("terms filter by modelIds:{}", detectModelIds); + log.info("terms filter by viewId:{}", viewIds); logTerms(terms); } return terms; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java index 252e9d210..3be851f87 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java @@ -1,12 +1,18 @@ package com.tencent.supersonic.chat.core.mapper; import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -14,11 +20,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; /** * DatabaseMatchStrategy uses SQL LIKE operator to match schema elements. @@ -59,7 +60,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy if (StringUtils.isBlank(detectSegment)) { return; } - Set modelIds = mapperHelper.getModelIds(queryContext.getModelId(), queryContext.getAgent()); + Set viewIds = mapperHelper.getViewIds(queryContext.getViewId(), queryContext.getAgent()); Double metricDimensionThresholdConfig = getThreshold(queryContext); @@ -72,9 +73,9 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy continue; } Set schemaElements = entry.getValue(); - if (!CollectionUtils.isEmpty(modelIds)) { + if (!CollectionUtils.isEmpty(viewIds)) { schemaElements = schemaElements.stream() - .filter(schemaElement -> modelIds.contains(schemaElement.getModel())) + .filter(schemaElement -> viewIds.contains(schemaElement.getView())) .collect(Collectors.toSet()); } for (SchemaElement schemaElement : schemaElements) { @@ -98,7 +99,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy Double metricDimensionThresholdConfig = optimizationConfig.getMetricDimensionThresholdConfig(); Double metricDimensionMinThresholdConfig = optimizationConfig.getMetricDimensionMinThresholdConfig(); - Map> modelElementMatches = queryContext.getMapInfo().getModelElementMatches(); + Map> modelElementMatches = queryContext.getMapInfo().getViewElementMatches(); boolean existElement = modelElementMatches.entrySet().stream().anyMatch(entry -> entry.getValue().size() >= 1); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java index bb3e1ed75..0e9fd87a9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java @@ -1,19 +1,20 @@ package com.tencent.supersonic.chat.core.mapper; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import java.util.List; -import java.util.stream.Collectors; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.stream.Collectors; + /** * A mapper capable of converting the VALUE of entity dimension values into ID types. */ @@ -23,12 +24,12 @@ public class EntityMapper extends BaseMapper { @Override public void doMap(QueryContext queryContext) { SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - for (Long modelId : schemaMapInfo.getMatchedModels()) { - List schemaElementMatchList = schemaMapInfo.getMatchedElements(modelId); + for (Long viewId : schemaMapInfo.getMatchedViewInfos()) { + List schemaElementMatchList = schemaMapInfo.getMatchedElements(viewId); if (CollectionUtils.isEmpty(schemaElementMatchList)) { continue; } - SchemaElement entity = getEntity(modelId, queryContext); + SchemaElement entity = getEntity(viewId, queryContext); if (entity == null || entity.getId() == null) { continue; } @@ -64,9 +65,9 @@ public class EntityMapper extends BaseMapper { return false; } - private SchemaElement getEntity(Long modelId, QueryContext queryContext) { + private SchemaElement getEntity(Long viewId, QueryContext queryContext) { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId); + ViewSchema modelSchema = semanticSchema.getViewSchemaMap().get(viewId); if (modelSchema != null && modelSchema.getEntity() != null) { return modelSchema.getEntity(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java index c5681c7f9..33aa4772e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java @@ -1,24 +1,25 @@ package com.tencent.supersonic.chat.core.mapper; import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult; import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.utils.HanlpHelper; import com.tencent.supersonic.chat.core.utils.NatureHelper; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /*** * A mapper that recognizes schema elements with keyword. @@ -56,8 +57,8 @@ public class KeywordMapper extends BaseMapper { for (HanlpMapResult hanlpMapResult : mapResults) { for (String nature : hanlpMapResult.getNatures()) { - Long modelId = NatureHelper.getModelId(nature); - if (Objects.isNull(modelId)) { + Long viewId = NatureHelper.getViewId(nature); + if (Objects.isNull(viewId)) { continue; } SchemaElementType elementType = NatureHelper.convertToElementType(nature); @@ -65,8 +66,8 @@ public class KeywordMapper extends BaseMapper { continue; } Long elementID = NatureHelper.getElementID(nature); - SchemaElement element = getSchemaElement(modelId, elementType, elementID, - queryContext.getSemanticSchema()); + SchemaElement element = getSchemaElement(viewId, elementType, + elementID, queryContext.getSemanticSchema()); if (element == null) { continue; } @@ -82,7 +83,7 @@ public class KeywordMapper extends BaseMapper { .detectWord(hanlpMapResult.getDetectWord()) .build(); - addToSchemaMap(queryContext.getMapInfo(), modelId, schemaElementMatch); + addToSchemaMap(queryContext.getMapInfo(), viewId, schemaElementMatch); } } } @@ -103,12 +104,12 @@ public class KeywordMapper extends BaseMapper { .similarity(mapperHelper.getSimilarity(match.getDetectWord(), schemaElement.getName())) .build(); log.info("add to schema, elementMatch {}", schemaElementMatch); - addToSchemaMap(queryContext.getMapInfo(), schemaElement.getModel(), schemaElementMatch); + addToSchemaMap(queryContext.getMapInfo(), schemaElement.getView(), schemaElementMatch); } } private Set getRegElementSet(SchemaMapInfo schemaMap, SchemaElement schemaElement) { - List elements = schemaMap.getMatchedElements(schemaElement.getModel()); + List elements = schemaMap.getMatchedElements(schemaElement.getView()); if (CollectionUtils.isEmpty(elements)) { return new HashSet<>(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java index f30a78a9e..50a25e4d6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java @@ -5,6 +5,11 @@ import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.chat.core.utils.NatureHelper; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -12,10 +17,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; @Data @Service @@ -61,7 +62,7 @@ public class MapperHelper { */ public boolean existDimensionValues(List natures) { for (String nature : natures) { - if (NatureHelper.isDimensionValueModelId(nature)) { + if (NatureHelper.isDimensionValueViewId(nature)) { return true; } } @@ -81,33 +82,33 @@ public class MapperHelper { detectSegment.length()); } - public Set getModelIds(Long modelId, Agent agent) { + public Set getViewIds(Long viewId, Agent agent) { - Set detectModelIds = new HashSet<>(); + Set detectViewIds = new HashSet<>(); if (Objects.nonNull(agent)) { - detectModelIds = agent.getModelIds(null); + detectViewIds = agent.getViewIds(null); } //contains all - if (Agent.containsAllModel(detectModelIds)) { - if (Objects.nonNull(modelId) && modelId > 0) { + if (Agent.containsAllModel(detectViewIds)) { + if (Objects.nonNull(viewId) && viewId > 0) { Set result = new HashSet<>(); - result.add(modelId); + result.add(viewId); return result; } return new HashSet<>(); } - if (Objects.nonNull(detectModelIds)) { - detectModelIds = detectModelIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); + if (Objects.nonNull(detectViewIds)) { + detectViewIds = detectViewIds.stream().filter(entry -> entry > 0).collect(Collectors.toSet()); } - if (Objects.nonNull(modelId) && modelId > 0 && Objects.nonNull(detectModelIds)) { - if (detectModelIds.contains(modelId)) { + if (Objects.nonNull(viewId) && viewId > 0 && Objects.nonNull(detectViewIds)) { + if (detectViewIds.contains(viewId)) { Set result = new HashSet<>(); - result.add(modelId); + result.add(viewId); return result; } } - return detectModelIds; + return detectViewIds; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelClusterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelClusterMapper.java deleted file mode 100644 index a67a4f0f0..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelClusterMapper.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.tencent.supersonic.chat.core.mapper; - -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.utils.ModelClusterBuilder; -import com.tencent.supersonic.common.pojo.ModelCluster; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/*** - * ModelClusterMapper build a cluster from - * connectable data models based on model-rela configuration - * and generate SchemaModelClusterMapInfo - */ -public class ModelClusterMapper implements SchemaMapper { - - @Override - public void map(QueryContext queryContext) { - SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - List modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema); - Map> modelClusterElementMatches = new HashMap<>(); - for (ModelCluster modelCluster : modelClusters) { - for (Long modelId : schemaMapInfo.getMatchedModels()) { - if (modelCluster.getModelIds().contains(modelId)) { - modelClusterElementMatches.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) - .addAll(schemaMapInfo.getMatchedElements(modelId)); - } - } - } - SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo(); - modelClusterMapInfo.setModelElementMatches(modelClusterElementMatches); - queryContext.setModelClusterMapInfo(modelClusterMapInfo); - } - - private List buildModelClusterMatched(SchemaMapInfo schemaMapInfo, - SemanticSchema semanticSchema) { - Set matchedModels = schemaMapInfo.getMatchedModels(); - List modelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema); - return modelClusters.stream().map(ModelCluster::getModelIds).peek(modelCluster -> { - modelCluster.removeIf(model -> !matchedModels.contains(model)); - }).filter(modelCluster -> modelCluster.size() > 0).map(ModelCluster::build).collect(Collectors.toList()); - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java index 9884a7577..d3042c00c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java @@ -10,11 +10,12 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; @Slf4j public class QueryFilterMapper implements SchemaMapper { @@ -23,22 +24,22 @@ public class QueryFilterMapper implements SchemaMapper { @Override public void map(QueryContext queryContext) { - Long modelId = queryContext.getModelId(); - if (modelId == null || modelId <= 0) { + Long viewId = queryContext.getViewId(); + if (viewId == null || viewId <= 0) { return; } SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - clearOtherSchemaElementMatch(modelId, schemaMapInfo); - List schemaElementMatches = schemaMapInfo.getMatchedElements(modelId); + clearOtherSchemaElementMatch(viewId, schemaMapInfo); + List schemaElementMatches = schemaMapInfo.getMatchedElements(viewId); if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); - schemaMapInfo.setMatchedElements(modelId, schemaElementMatches); + schemaMapInfo.setMatchedElements(viewId, schemaElementMatches); } addValueSchemaElementMatch(queryContext, schemaElementMatches); } private void clearOtherSchemaElementMatch(Long modelId, SchemaMapInfo schemaMapInfo) { - for (Map.Entry> entry : schemaMapInfo.getModelElementMatches().entrySet()) { + for (Map.Entry> entry : schemaMapInfo.getViewElementMatches().entrySet()) { if (!entry.getKey().equals(modelId)) { entry.getValue().clear(); } @@ -60,7 +61,7 @@ public class QueryFilterMapper implements SchemaMapper { .name(String.valueOf(filter.getValue())) .type(SchemaElementType.VALUE) .bizName(filter.getBizName()) - .model(queryContext.getModelId()) + .view(queryContext.getViewId()) .build(); SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() .element(element) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java index 8019fa741..a3f8f61e0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/JavaLLMProxy.java @@ -1,23 +1,24 @@ package com.tencent.supersonic.chat.core.parser; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionPromptGenerator; +import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; +import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat; import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGeneration; import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGenerationFactory; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; -import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.ContextUtils; import dev.langchain4j.model.chat.ChatLanguageModel; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import java.util.Objects; + /** * LLMProxy based on langchain4j Java version. */ @@ -37,12 +38,12 @@ public class JavaLLMProxy implements LLMProxy { return false; } - public LLMResp query2sql(LLMReq llmReq, String modelClusterKey) { + public LLMResp query2sql(LLMReq llmReq, Long viewId) { SqlGeneration sqlGeneration = SqlGenerationFactory.get( SqlGenerationMode.getMode(llmReq.getSqlGenerationMode())); - String modelName = llmReq.getSchema().getModelName(); - LLMResp result = sqlGeneration.generation(llmReq, modelClusterKey); + String modelName = llmReq.getSchema().getViewName(); + LLMResp result = sqlGeneration.generation(llmReq, viewId); result.setQuery(llmReq.getQueryText()); result.setModelName(modelName); return result; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java index 5f8049200..0f47cf962 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/LLMProxy.java @@ -15,7 +15,7 @@ public interface LLMProxy { boolean isSkip(QueryContext queryContext); - LLMResp query2sql(LLMReq llmReq, String modelClusterKey); + LLMResp query2sql(LLMReq llmReq, Long viewId); FunctionResp requestFunction(FunctionReq functionReq); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java index a420f8bc5..2cf77966f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/PythonLLMProxy.java @@ -1,19 +1,16 @@ package com.tencent.supersonic.chat.core.parser; import com.alibaba.fastjson.JSON; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.config.LLMParserConfig; -import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat; import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionCallConfig; import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq; import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp; +import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -28,6 +25,10 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; + /** * PythonLLMProxy sends requests to LangChain-based python service. */ @@ -47,10 +48,10 @@ public class PythonLLMProxy implements LLMProxy { return false; } - public LLMResp query2sql(LLMReq llmReq, String modelClusterKey) { + public LLMResp query2sql(LLMReq llmReq, Long viewId) { long startTime = System.currentTimeMillis(); - log.info("requestLLM request, modelId:{},llmReq:{}", modelClusterKey, llmReq); - keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); + log.info("requestLLM request, viewId:{},llmReq:{}", viewId, llmReq); + keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); try { LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java index 2ea3cc1af..3c1893faf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java @@ -13,13 +13,14 @@ import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; /** * QueryTypeParser resolves query type as either METRIC or TAG, or ID. @@ -49,7 +50,7 @@ public class QueryTypeParser implements SemanticParser { return QueryType.ID; } //1. entity queryType - Set modelIds = parseInfo.getModel().getModelIds(); + Long viewId = parseInfo.getViewId(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) { //If all the fields in the SELECT statement are of tag type. @@ -58,12 +59,12 @@ public class QueryTypeParser implements SemanticParser { .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(whereFields)) { - Set ids = semanticSchema.getEntities(modelIds).stream().map(SchemaElement::getName) + Set ids = semanticSchema.getEntities(viewId).stream().map(SchemaElement::getName) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(ids) && ids.stream().anyMatch(whereFields::contains)) { return QueryType.ID; } - Set tags = semanticSchema.getTags(modelIds).stream().map(SchemaElement::getName) + Set tags = semanticSchema.getTags(viewId).stream().map(SchemaElement::getName) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(tags) && tags.containsAll(whereFields)) { return QueryType.TAG; @@ -72,7 +73,7 @@ public class QueryTypeParser implements SemanticParser { } //2. metric queryType List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL()); - List metrics = semanticSchema.getMetrics(modelIds); + List metrics = semanticSchema.getMetrics(viewId); if (CollectionUtils.isNotEmpty(metrics)) { Set metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); boolean containMetric = selectFields.stream().anyMatch(metricNameSet::contains); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java index 35e935cde..6d034e5b5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java @@ -18,7 +18,6 @@ import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import org.springframework.util.CollectionUtils; import java.util.HashMap; @@ -56,13 +55,13 @@ public abstract class PluginParser implements SemanticParser { public void buildQuery(QueryContext queryContext, PluginRecallResult pluginRecallResult) { Plugin plugin = pluginRecallResult.getPlugin(); - Set modelIds = pluginRecallResult.getModelIds(); + Set viewIds = pluginRecallResult.getViewIds(); if (plugin.isContainsAllModel()) { - modelIds = Sets.newHashSet(-1L); + viewIds = Sets.newHashSet(-1L); } - for (Long modelId : modelIds) { + for (Long viewId : viewIds) { PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); - SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, + SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(viewId, plugin, queryContext, pluginRecallResult.getDistance()); semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); semanticParseInfo.setScore(pluginRecallResult.getScore()); @@ -75,20 +74,19 @@ public abstract class PluginParser implements SemanticParser { return PluginManager.getPluginAgentCanSupport(queryContext); } - protected SemanticParseInfo buildSemanticParseInfo(Long modelId, Plugin plugin, + protected SemanticParseInfo buildSemanticParseInfo(Long viewId, Plugin plugin, QueryContext queryContext, double distance) { - List schemaElementMatches = - queryContext.getModelClusterMapInfo().getMatchedElements(modelId); + List schemaElementMatches = queryContext.getMapInfo().getMatchedElements(viewId); QueryFilters queryFilters = queryContext.getQueryFilters(); - if (modelId == null && !CollectionUtils.isEmpty(plugin.getModelList())) { - modelId = plugin.getModelList().get(0); + if (viewId == null && !CollectionUtils.isEmpty(plugin.getViewList())) { + viewId = plugin.getViewList().get(0); } if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); } SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setElementMatches(schemaElementMatches); - semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(modelId))); + semanticParseInfo.setView(queryContext.getSemanticSchema().getView(viewId)); Map properties = new HashMap<>(); PluginParseResult pluginParseResult = new PluginParseResult(); pluginParseResult.setPlugin(plugin); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java index c6f2f351e..f0b7f1b4a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/embedding/EmbeddingRecallParser.java @@ -57,15 +57,15 @@ public class EmbeddingRecallParser extends PluginParser { Pair> pair = PluginManager.resolve(plugin, queryContext); log.info("embedding plugin resolve: {}", pair); if (pair.getLeft()) { - Set modelList = pair.getRight(); - if (CollectionUtils.isEmpty(modelList)) { + Set viewList = pair.getRight(); + if (CollectionUtils.isEmpty(viewList)) { continue; } plugin.setParseMode(ParseMode.EMBEDDING_RECALL); double distance = embeddingRetrieval.getDistance(); double score = queryContext.getQueryText().length() * (1 - distance); return PluginRecallResult.builder() - .plugin(plugin).modelIds(modelList).score(score).distance(distance).build(); + .plugin(plugin).viewIds(viewList).score(score).distance(distance).build(); } } return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java index 5c8a1e3c6..69f39ea73 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/function/FunctionCallParser.java @@ -12,15 +12,16 @@ import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + /** * FunctionCallParser is an implementation of a recall plugin based on FunctionCall */ @@ -56,19 +57,19 @@ public class FunctionCallParser extends PluginParser { plugin.setParseMode(ParseMode.FUNCTION_CALL); Pair> pluginResolveResult = PluginManager.resolve(plugin, queryContext); if (pluginResolveResult.getLeft()) { - Set modelList = pluginResolveResult.getRight(); - if (CollectionUtils.isEmpty(modelList)) { + Set viewList = pluginResolveResult.getRight(); + if (CollectionUtils.isEmpty(viewList)) { return null; } double score = queryContext.getQueryText().length(); - return PluginRecallResult.builder().plugin(plugin).modelIds(modelList).score(score).build(); + return PluginRecallResult.builder().plugin(plugin).viewIds(viewList).score(score).build(); } return null; } public FunctionResp functionCall(QueryContext queryContext) { List pluginToFunctionCall = - getPluginToFunctionCall(queryContext.getModelId(), queryContext); + getPluginToFunctionCall(queryContext.getViewId(), queryContext); if (CollectionUtils.isEmpty(pluginToFunctionCall)) { log.info("function call parser, plugin is empty, skip"); return null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicModelResolver.java deleted file mode 100644 index 47671a3ca..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicModelResolver.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; - -import com.tencent.supersonic.chat.core.query.SemanticQuery; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; -import com.tencent.supersonic.common.pojo.ModelCluster; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class HeuristicModelResolver implements ModelResolver { - - protected static String selectModelBySchemaElementMatchScore(Map modelQueryModes, - SchemaModelClusterMapInfo schemaMap) { - //model count priority - String modelIdByModelCount = getModelIdByMatchModelScore(schemaMap); - if (Objects.nonNull(modelIdByModelCount)) { - log.info("selectModel by model count:{}", modelIdByModelCount); - return modelIdByModelCount; - } - - Map modelTypeMap = getModelTypeMap(schemaMap); - if (modelTypeMap.size() == 1) { - String modelSelect = modelTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey(); - if (modelQueryModes.containsKey(modelSelect)) { - log.info("selectModel with only one Model [{}]", modelSelect); - return modelSelect; - } - } else { - Map.Entry maxModel = modelTypeMap.entrySet().stream() - .filter(entry -> modelQueryModes.containsKey(entry.getKey())) - .sorted((o1, o2) -> { - int difference = o2.getValue().getCount() - o1.getValue().getCount(); - if (difference == 0) { - return (int) ((o2.getValue().getMaxSimilarity() - - o1.getValue().getMaxSimilarity()) * 100); - } - return difference; - }).findFirst().orElse(null); - if (maxModel != null) { - log.info("selectModel with multiple Models [{}]", maxModel.getKey()); - return maxModel.getKey(); - } - } - return null; - } - - private static String getModelIdByMatchModelScore(SchemaModelClusterMapInfo schemaMap) { - Map> modelElementMatches = schemaMap.getModelElementMatches(); - // calculate model match score, matched element gets 1.0 point, and inherit element gets 0.5 point - Map modelIdToModelScore = new HashMap<>(); - if (Objects.nonNull(modelElementMatches)) { - for (Entry> modelElementMatch : modelElementMatches.entrySet()) { - String modelId = modelElementMatch.getKey(); - List modelMatchesScore = modelElementMatch.getValue().stream() - .filter(elementMatch -> elementMatch.getSimilarity() >= 1) - .filter(elementMatch -> SchemaElementType.MODEL.equals(elementMatch.getElement().getType())) - .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0).collect(Collectors.toList()); - - if (!CollectionUtils.isEmpty(modelMatchesScore)) { - // get sum of model match score - double score = modelMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); - modelIdToModelScore.put(modelId, score); - } - } - Entry maxModelScore = modelIdToModelScore.entrySet().stream() - .max(Comparator.comparingDouble(o -> o.getValue())).orElse(null); - log.info("maxModelCount:{},modelIdToModelCount:{}", maxModelScore, modelIdToModelScore); - if (Objects.nonNull(maxModelScore)) { - return maxModelScore.getKey(); - } - } - return null; - } - - public static Map getModelTypeMap(SchemaModelClusterMapInfo schemaMap) { - Map modelCount = new HashMap<>(); - for (Map.Entry> entry : schemaMap.getModelElementMatches().entrySet()) { - List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); - if (schemaElementMatches != null && schemaElementMatches.size() > 0) { - if (!modelCount.containsKey(entry.getKey())) { - modelCount.put(entry.getKey(), new ModelMatchResult()); - } - ModelMatchResult modelMatchResult = modelCount.get(entry.getKey()); - Set schemaElementTypes = new HashSet<>(); - schemaElementMatches.stream() - .forEach(schemaElementMatch -> schemaElementTypes.add( - schemaElementMatch.getElement().getType())); - SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() - .sorted((o1, o2) -> - ((int) ((o2.getSimilarity() - o1.getSimilarity()) * 100)) - ).findFirst().orElse(null); - if (schemaElementMatchMax != null) { - modelMatchResult.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); - } - modelMatchResult.setCount(schemaElementTypes.size()); - - } - } - return modelCount; - } - - public String resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels) { - SchemaModelClusterMapInfo mapInfo = queryContext.getModelClusterMapInfo(); - Set matchedModelClusters = mapInfo.getElementMatchesByModelIds(restrictiveModels).keySet(); - Long modelId = queryContext.getModelId(); - if (Objects.nonNull(modelId) && modelId > 0) { - if (CollectionUtils.isEmpty(restrictiveModels) || restrictiveModels.contains(modelId)) { - return getModelClusterByModelId(modelId, matchedModelClusters); - } - return null; - } - - Map modelQueryModes = new HashMap<>(); - for (String matchedModel : matchedModelClusters) { - modelQueryModes.put(matchedModel, null); - } - if (modelQueryModes.size() == 1) { - return modelQueryModes.keySet().stream().findFirst().get(); - } - return selectModelBySchemaElementMatchScore(modelQueryModes, mapInfo); - } - - private String getModelClusterByModelId(Long modelId, Set modelClusterKeySet) { - for (String modelClusterKey : modelClusterKeySet) { - if (ModelCluster.build(modelClusterKey).getModelIds().contains(modelId)) { - return modelClusterKey; - } - } - return null; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java new file mode 100644 index 000000000..591511fa2 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java @@ -0,0 +1,136 @@ +package com.tencent.supersonic.chat.core.parser.sql.llm; + +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.chat.core.query.SemanticQuery; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class HeuristicViewResolver implements ViewResolver { + + protected static Long selectViewBySchemaElementMatchScore(Map viewQueryModes, + SchemaMapInfo schemaMap) { + //view count priority + Long viewIdByViewCount = getViewIdByMatchViewScore(schemaMap); + if (Objects.nonNull(viewIdByViewCount)) { + log.info("selectView by view count:{}", viewIdByViewCount); + return viewIdByViewCount; + } + + Map viewTypeMap = getViewTypeMap(schemaMap); + if (viewTypeMap.size() == 1) { + Long viewSelect = new ArrayList<>(viewTypeMap.entrySet()).get(0).getKey(); + if (viewQueryModes.containsKey(viewSelect)) { + log.info("selectView with only one View [{}]", viewSelect); + return viewSelect; + } + } else { + Map.Entry maxView = viewTypeMap.entrySet().stream() + .filter(entry -> viewQueryModes.containsKey(entry.getKey())) + .sorted((o1, o2) -> { + int difference = o2.getValue().getCount() - o1.getValue().getCount(); + if (difference == 0) { + return (int) ((o2.getValue().getMaxSimilarity() + - o1.getValue().getMaxSimilarity()) * 100); + } + return difference; + }).findFirst().orElse(null); + if (maxView != null) { + log.info("selectView with multiple Views [{}]", maxView.getKey()); + return maxView.getKey(); + } + } + return null; + } + + private static Long getViewIdByMatchViewScore(SchemaMapInfo schemaMap) { + Map> viewElementMatches = schemaMap.getViewElementMatches(); + // calculate view match score, matched element gets 1.0 point, and inherit element gets 0.5 point + Map viewIdToViewScore = new HashMap<>(); + if (Objects.nonNull(viewElementMatches)) { + for (Entry> viewElementMatch : viewElementMatches.entrySet()) { + Long viewId = viewElementMatch.getKey(); + List viewMatchesScore = viewElementMatch.getValue().stream() + .filter(elementMatch -> elementMatch.getSimilarity() >= 1) + .filter(elementMatch -> SchemaElementType.VIEW.equals(elementMatch.getElement().getType())) + .map(elementMatch -> elementMatch.isInherited() ? 0.5 : 1.0).collect(Collectors.toList()); + + if (!CollectionUtils.isEmpty(viewMatchesScore)) { + // get sum of view match score + double score = viewMatchesScore.stream().mapToDouble(Double::doubleValue).sum(); + viewIdToViewScore.put(viewId, score); + } + } + Entry maxViewScore = viewIdToViewScore.entrySet().stream() + .max(Comparator.comparingDouble(Entry::getValue)).orElse(null); + log.info("maxViewCount:{},viewIdToViewCount:{}", maxViewScore, viewIdToViewScore); + if (Objects.nonNull(maxViewScore)) { + return maxViewScore.getKey(); + } + } + return null; + } + + public static Map getViewTypeMap(SchemaMapInfo schemaMap) { + Map viewCount = new HashMap<>(); + for (Map.Entry> entry : schemaMap.getViewElementMatches().entrySet()) { + List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); + if (schemaElementMatches != null && schemaElementMatches.size() > 0) { + if (!viewCount.containsKey(entry.getKey())) { + viewCount.put(entry.getKey(), new ViewMatchResult()); + } + ViewMatchResult viewMatchResult = viewCount.get(entry.getKey()); + Set schemaElementTypes = new HashSet<>(); + schemaElementMatches.stream() + .forEach(schemaElementMatch -> schemaElementTypes.add( + schemaElementMatch.getElement().getType())); + SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() + .sorted((o1, o2) -> + ((int) ((o2.getSimilarity() - o1.getSimilarity()) * 100)) + ).findFirst().orElse(null); + if (schemaElementMatchMax != null) { + viewMatchResult.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); + } + viewMatchResult.setCount(schemaElementTypes.size()); + + } + } + return viewCount; + } + + public Long resolve(QueryContext queryContext, Set agentViewIds) { + SchemaMapInfo mapInfo = queryContext.getMapInfo(); + Set matchedViews = mapInfo.getMatchedViewInfos(); + Long viewId = queryContext.getViewId(); + if (Objects.nonNull(viewId) && viewId > 0) { + if (CollectionUtils.isEmpty(agentViewIds) || agentViewIds.contains(viewId)) { + return viewId; + } + return null; + } + matchedViews.retainAll(agentViewIds); + Map viewQueryModes = new HashMap<>(); + for (Long viewIds : matchedViews) { + viewQueryModes.put(viewIds, null); + } + if (viewQueryModes.size() == 1) { + return viewQueryModes.keySet().stream().findFirst().get(); + } + return selectViewBySchemaElementMatchScore(viewQueryModes, mapInfo); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java index 945f50e23..abc2c7507 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; @@ -11,21 +12,23 @@ import com.tencent.supersonic.chat.core.config.LLMParserConfig; import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.parser.SatisfactionChecker; -import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -35,12 +38,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Slf4j @Service @@ -63,42 +60,20 @@ public class LLMRequestService { return false; } - public ModelCluster getModelCluster(QueryContext queryCtx, ChatContext chatCtx) { + public Long getViewId(QueryContext queryCtx) { Agent agent = queryCtx.getAgent(); - Set distinctModelIds = new HashSet<>(); + Set agentViewIds = new HashSet<>(); if (Objects.nonNull(agent)) { - distinctModelIds = agent.getModelIds(AgentToolType.NL2SQL_LLM); + agentViewIds = agent.getViewIds(AgentToolType.NL2SQL_LLM); } - if (llmParserConfig.getAllModel()) { - ModelCluster modelCluster = ModelCluster.build(distinctModelIds); - if (!CollectionUtils.isEmpty(queryCtx.getCandidateQueries())) { - queryCtx.getCandidateQueries().stream().forEach(o -> { - if (LLMSqlQuery.QUERY_MODE.equals(o.getParseInfo().getQueryMode())) { - o.getParseInfo().setModel(modelCluster); - } - }); - } - SemanticQuery semanticQuery = QueryManager.createQuery(LLMSqlQuery.QUERY_MODE); - semanticQuery.getParseInfo().setModel(modelCluster); - List schemaElementMatches = new ArrayList<>(); - distinctModelIds.stream().forEach(o -> { - if (!CollectionUtils.isEmpty(queryCtx.getMapInfo().getMatchedElements(o))) { - schemaElementMatches.addAll(queryCtx.getMapInfo().getMatchedElements(o)); - } - }); - queryCtx.getModelClusterMapInfo().setMatchedElements(modelCluster.getKey(), schemaElementMatches); - return modelCluster; + if (Agent.containsAllModel(agentViewIds)) { + agentViewIds = new HashSet<>(); } - if (Agent.containsAllModel(distinctModelIds)) { - distinctModelIds = new HashSet<>(); - } - ModelResolver modelResolver = ComponentFactory.getModelResolver(); - String modelCluster = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds); - log.info("resolve modelId:{},llmParser Models:{}", modelCluster, distinctModelIds); - return ModelCluster.build(modelCluster); + ViewResolver viewResolver = ComponentFactory.getModelResolver(); + return viewResolver.resolve(queryCtx, agentViewIds); } - public NL2SQLTool getParserTool(QueryContext queryCtx, Set modelIdSet) { + public NL2SQLTool getParserTool(QueryContext queryCtx, Long viewId) { Agent agent = queryCtx.getAgent(); if (Objects.isNull(agent)) { return null; @@ -106,39 +81,33 @@ public class LLMRequestService { List commonAgentTools = agent.getParserTools(AgentToolType.NL2SQL_LLM); Optional llmParserTool = commonAgentTools.stream() .filter(tool -> { - List modelIds = tool.getModelIds(); - if (Agent.containsAllModel(new HashSet<>(modelIds))) { + List viewIds = tool.getViewIds(); + if (Agent.containsAllModel(new HashSet<>(viewIds))) { return true; } - for (Long modelId : modelIdSet) { - if (modelIds.contains(modelId)) { - return true; - } - } - return false; + return viewIds.contains(viewId); }) .findFirst(); return llmParserTool.orElse(null); } - public LLMReq getLlmReq(QueryContext queryCtx, SemanticSchema semanticSchema, - ModelCluster modelCluster, List linkingValues) { - Map modelIdToName = semanticSchema.getModelIdToName(); + public LLMReq getLlmReq(QueryContext queryCtx, Long viewId, + SemanticSchema semanticSchema, List linkingValues) { + Map viewIdToName = semanticSchema.getViewIdToName(); String queryText = queryCtx.getQueryText(); LLMReq llmReq = new LLMReq(); llmReq.setQueryText(queryText); - Long firstModelId = modelCluster.getFirstModel(); LLMReq.FilterCondition filterCondition = new LLMReq.FilterCondition(); llmReq.setFilterCondition(filterCondition); LLMReq.LLMSchema llmSchema = new LLMReq.LLMSchema(); - llmSchema.setModelName(modelIdToName.get(firstModelId)); - llmSchema.setDomainName(modelIdToName.get(firstModelId)); + llmSchema.setViewName(viewIdToName.get(viewId)); + llmSchema.setDomainName(viewIdToName.get(viewId)); - List fieldNameList = getFieldNameList(queryCtx, modelCluster, llmParserConfig); + List fieldNameList = getFieldNameList(queryCtx, viewId, llmParserConfig); - String priorExts = getPriorExts(modelCluster.getModelIds(), fieldNameList); + String priorExts = getPriorExts(viewId, fieldNameList); llmReq.setPriorExts(priorExts); fieldNameList.add(TimeDimensionEnum.DAY.getChName()); @@ -151,7 +120,7 @@ public class LLMRequestService { } llmReq.setLinking(linking); - String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, firstModelId); + String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, viewId); if (StringUtils.isEmpty(currentDate)) { currentDate = DateUtils.getBeforeDate(0); } @@ -160,29 +129,28 @@ public class LLMRequestService { return llmReq; } - public LLMResp requestLLM(LLMReq llmReq, String modelClusterKey) { - return ComponentFactory.getLLMProxy().query2sql(llmReq, modelClusterKey); + public LLMResp requestLLM(LLMReq llmReq, Long viewId) { + return ComponentFactory.getLLMProxy().query2sql(llmReq, viewId); } - protected List getFieldNameList(QueryContext queryCtx, ModelCluster modelCluster, + protected List getFieldNameList(QueryContext queryCtx, Long viewId, LLMParserConfig llmParserConfig) { - Set results = getTopNFieldNames(queryCtx, modelCluster, llmParserConfig); + Set results = getTopNFieldNames(queryCtx, viewId, llmParserConfig); - Set fieldNameList = getMatchedFieldNames(queryCtx, modelCluster); + Set fieldNameList = getMatchedFieldNames(queryCtx, viewId); results.addAll(fieldNameList); return new ArrayList<>(results); } - private String getPriorExts(Set modelIds, List fieldNameList) { + private String getPriorExts(Long viewId, List fieldNameList) { StringBuilder extraInfoSb = new StringBuilder(); - List modelSchemaResps = semanticInterpreter.fetchModelSchema( - new ArrayList<>(modelIds), true); - if (!CollectionUtils.isEmpty(modelSchemaResps)) { - - ModelSchemaResp modelSchemaResp = modelSchemaResps.get(0); - Map fieldNameToDataFormatType = modelSchemaResp.getMetrics() + List viewSchemaResps = semanticInterpreter.fetchViewSchema( + Lists.newArrayList(viewId), true); + if (!CollectionUtils.isEmpty(viewSchemaResps)) { + ViewSchemaResp viewSchemaResp = viewSchemaResps.get(0); + Map fieldNameToDataFormatType = viewSchemaResp.getMetrics() .stream().filter(metricSchemaResp -> Objects.nonNull(metricSchemaResp.getDataFormatType())) .flatMap(metricSchemaResp -> { Set> result = new HashSet<>(); @@ -210,11 +178,9 @@ public class LLMRequestService { return extraInfoSb.toString(); } - protected List getValueList(QueryContext queryCtx, ModelCluster modelCluster) { - Map itemIdToName = getItemIdToName(queryCtx, modelCluster); - - List matchedElements = queryCtx.getModelClusterMapInfo() - .getMatchedElements(modelCluster.getKey()); + protected List getValueList(QueryContext queryCtx, Long viewId) { + Map itemIdToName = getItemIdToName(queryCtx, viewId); + List matchedElements = queryCtx.getMapInfo().getMatchedElements(viewId); if (CollectionUtils.isEmpty(matchedElements)) { return new ArrayList<>(); } @@ -234,22 +200,21 @@ public class LLMRequestService { return new ArrayList<>(valueMatches); } - protected Map getItemIdToName(QueryContext queryCtx, ModelCluster modelCluster) { + protected Map getItemIdToName(QueryContext queryCtx, Long viewId) { SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - return semanticSchema.getDimensions(modelCluster.getModelIds()).stream() + return semanticSchema.getDimensions(viewId).stream() .collect(Collectors.toMap(SchemaElement::getId, SchemaElement::getName, (value1, value2) -> value2)); } - private Set getTopNFieldNames(QueryContext queryCtx, ModelCluster modelCluster, - LLMParserConfig llmParserConfig) { + private Set getTopNFieldNames(QueryContext queryCtx, Long viewId, LLMParserConfig llmParserConfig) { SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - Set results = semanticSchema.getDimensions(modelCluster.getModelIds()).stream() + Set results = semanticSchema.getDimensions(viewId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getDimensionTopN()) .map(entry -> entry.getName()) .collect(Collectors.toSet()); - Set metrics = semanticSchema.getMetrics(modelCluster.getModelIds()).stream() + Set metrics = semanticSchema.getMetrics(viewId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getMetricTopN()) .map(entry -> entry.getName()) @@ -259,10 +224,9 @@ public class LLMRequestService { return results; } - protected Set getMatchedFieldNames(QueryContext queryCtx, ModelCluster modelCluster) { - Map itemIdToName = getItemIdToName(queryCtx, modelCluster); - List matchedElements = queryCtx.getModelClusterMapInfo() - .getMatchedElements(modelCluster.getKey()); + protected Set getMatchedFieldNames(QueryContext queryCtx, Long viewId) { + Map itemIdToName = getItemIdToName(queryCtx, viewId); + List matchedElements = queryCtx.getMapInfo().getMatchedElements(viewId); if (CollectionUtils.isEmpty(matchedElements)) { return new HashSet<>(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java index 4f3db2723..15f4c5714 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java @@ -28,10 +28,9 @@ public class LLMResponseService { } LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); - parseInfo.setModel(parseResult.getModelCluster()); + parseInfo.setView(queryCtx.getSemanticSchema().getView(parseResult.getViewId())); NL2SQLTool commonAgentTool = parseResult.getCommonAgentTool(); - parseInfo.getElementMatches().addAll(queryCtx.getModelClusterMapInfo() - .getMatchedElements(parseInfo.getModelClusterKey())); + parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getViewId())); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, parseResult); @@ -42,7 +41,6 @@ public class LLMResponseService { parseInfo.setScore(queryCtx.getQueryText().length() * (1 + weight)); parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.getSqlInfo().setS2SQL(s2SQL); - parseInfo.setModel(parseResult.getModelCluster()); queryCtx.getCandidateQueries().add(semanticQuery); return parseInfo; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java index 7a65df0ef..d415b9473 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java @@ -9,14 +9,13 @@ import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; + import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; @Slf4j public class LLMSqlParser implements SemanticParser { @@ -30,31 +29,30 @@ public class LLMSqlParser implements SemanticParser { } try { //2.get modelId from queryCtx and chatCtx. - ModelCluster modelCluster = requestService.getModelCluster(queryCtx, chatCtx); - if (StringUtils.isBlank(modelCluster.getKey())) { + Long viewId = requestService.getViewId(queryCtx); + if (viewId == null) { return; } //3.get agent tool and determine whether to skip this parser. - NL2SQLTool commonAgentTool = requestService.getParserTool(queryCtx, modelCluster.getModelIds()); + NL2SQLTool commonAgentTool = requestService.getParserTool(queryCtx, viewId); if (Objects.isNull(commonAgentTool)) { log.info("no tool in this agent, skip {}", LLMSqlParser.class); return; } //4.construct a request, call the API for the large model, and retrieve the results. - List linkingValues = requestService.getValueList(queryCtx, modelCluster); + List linkingValues = requestService.getValueList(queryCtx, viewId); SemanticSchema semanticSchema = queryCtx.getSemanticSchema(); - LLMReq llmReq = requestService.getLlmReq(queryCtx, semanticSchema, modelCluster, linkingValues); - LLMResp llmResp = requestService.requestLLM(llmReq, modelCluster.getKey()); + LLMReq llmReq = requestService.getLlmReq(queryCtx, viewId, semanticSchema, linkingValues); + LLMResp llmResp = requestService.requestLLM(llmReq, viewId); if (Objects.isNull(llmResp)) { return; } //5. deduplicate the SQL result list and build parserInfo - modelCluster.buildName(semanticSchema.getModelIdToName()); LLMResponseService responseService = ContextUtils.getBean(LLMResponseService.class); Map deduplicationSqlResp = responseService.getDeduplicationSqlResp(llmResp); ParseResult parseResult = ParseResult.builder() - .modelCluster(modelCluster) + .viewId(viewId) .commonAgentTool(commonAgentTool) .llmReq(llmReq) .llmResp(llmResp) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelResolver.java deleted file mode 100644 index 840e7475d..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelResolver.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.tencent.supersonic.chat.core.parser.sql.llm; - - -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import java.util.Set; - -public interface ModelResolver { - - String resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels); - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java index 1f58d5987..13eeee5fa 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSCSqlGeneration.java @@ -11,11 +11,6 @@ import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; @@ -24,6 +19,12 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; + @Service @Slf4j public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { @@ -42,9 +43,9 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, String modelClusterKey) { + public LLMResp generation(LLMReq llmReq, Long viewId) { //1.retriever sqlExamples and generate exampleListPool - keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); + keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java index 949e8028a..a3fe2748f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/OnePassSqlGeneration.java @@ -2,19 +2,16 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.tencent.supersonic.chat.core.config.OptimizationConfig; -import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.common.util.JsonUtil; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +19,10 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @Service @Slf4j public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { @@ -31,7 +32,7 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { private ChatLanguageModel chatLanguageModel; @Autowired - private SqlExamplarLoader sqlExamplarLoader; + private SqlExamplarLoader sqlExampleLoader; @Autowired private OptimizationConfig optimizationConfig; @@ -40,10 +41,10 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, String modelClusterKey) { + public LLMResp generation(LLMReq llmReq, Long viewId) { //1.retriever sqlExamples - keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); - List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), + keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); + List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); //2.generator linking and sql prompt by sqlExamples,and generate response. diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java index c058d0e24..ef323bde0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ParseResult.java @@ -5,7 +5,6 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; -import com.tencent.supersonic.common.pojo.ModelCluster; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -19,7 +18,7 @@ import java.util.List; @NoArgsConstructor public class ParseResult { - private ModelCluster modelCluster; + private Long viewId; private LLMReq llmReq; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java index 638d99e57..e897e76e0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlGeneration.java @@ -12,9 +12,9 @@ public interface SqlGeneration { /*** * generate llmResp (sql, schemaLink, prompt, etc.) through LLMReq. * @param llmReq - * @param modelClusterKey + * @param viewId * @return */ - LLMResp generation(LLMReq llmReq, String modelClusterKey); + LLMResp generation(LLMReq llmReq, Long viewId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java index 0781cb098..155ef0655 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/SqlPromptGenerator.java @@ -2,14 +2,15 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.stereotype.Component; @Component @Slf4j @@ -95,7 +96,7 @@ public class SqlPromptGenerator { } public Pair transformQuestionPrompt(LLMReq llmReq) { - String modelName = llmReq.getSchema().getModelName(); + String modelName = llmReq.getSchema().getViewName(); List fieldNameList = llmReq.getSchema().getFieldNameList(); List linking = llmReq.getLinking(); String currentDate = llmReq.getCurrentDate(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java index 9e5e7e992..3ba1452b5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSCSqlGeneration.java @@ -11,10 +11,6 @@ import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +18,11 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + @Service public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { @@ -39,9 +40,9 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, String modelClusterKey) { + public LLMResp generation(LLMReq llmReq, Long viewId) { //1.retriever sqlExamples and generate exampleListPool - keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); + keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java index 3d0549db9..aaaf3eb68 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/TwoPassSqlGeneration.java @@ -11,9 +11,6 @@ import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +18,10 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @Service @Slf4j public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { @@ -39,8 +40,8 @@ public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { private SqlPromptGenerator sqlPromptGenerator; @Override - public LLMResp generation(LLMReq llmReq, String modelClusterKey) { - keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); + public LLMResp generation(LLMReq llmReq, Long viewId) { + keyPipelineLog.info("viewId:{},llmReq:{}", viewId, llmReq); List> sqlExamples = sqlExamplarLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelMatchResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java similarity index 82% rename from chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelMatchResult.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java index 70eb3d68c..7d3acaba0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ModelMatchResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewMatchResult.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; import lombok.Data; @Data -public class ModelMatchResult { +public class ViewMatchResult { private Integer count = 0; private double maxSimilarity; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java new file mode 100644 index 000000000..e3bb72f20 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/ViewResolver.java @@ -0,0 +1,12 @@ +package com.tencent.supersonic.chat.core.parser.sql.llm; + + +import com.tencent.supersonic.chat.core.pojo.QueryContext; + +import java.util.Set; + +public interface ViewResolver { + + Long resolve(QueryContext queryContext, Set restrictiveModels); + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java index d078a6e3e..9031c6fb5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/AgentCheckParser.java @@ -11,12 +11,12 @@ import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.common.pojo.enums.QueryType; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import java.util.List; +import java.util.stream.Collectors; + @Slf4j public class AgentCheckParser implements SemanticParser { @@ -52,16 +52,13 @@ public class AgentCheckParser implements SemanticParser { return !tool.getQueryTypes().contains(QueryType.METRIC.name()); } } - if (CollectionUtils.isEmpty(tool.getModelIds())) { + if (CollectionUtils.isEmpty(tool.getViewIds())) { return true; } if (tool.isContainsAllModel()) { return false; } - if (new HashSet<>(tool.getModelIds()) - .containsAll(query.getParseInfo().getModel().getModelIds())) { - return false; - } + return !tool.getViewIds().contains(query.getParseInfo().getViewId()); } return true; }); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java index 8bdc8239c..1097af9b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.core.parser.sql.rule; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; @@ -12,8 +11,8 @@ import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.core.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.core.query.rule.metric.MetricSemanticQuery; import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery; -import com.tencent.supersonic.chat.core.utils.ModelClusterBuilder; -import com.tencent.supersonic.common.pojo.ModelCluster; +import lombok.extern.slf4j.Slf4j; + import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; @@ -23,8 +22,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; /** * ContextInheritParser tries to inherit certain schema elements from context @@ -42,7 +39,7 @@ public class ContextInheritParser implements SemanticParser { SchemaElementType.VALUE, Arrays.asList(SchemaElementType.VALUE, SchemaElementType.DIMENSION)), new AbstractMap.SimpleEntry<>(SchemaElementType.ENTITY, Arrays.asList(SchemaElementType.ENTITY)), new AbstractMap.SimpleEntry<>(SchemaElementType.TAG, Arrays.asList(SchemaElementType.TAG)), - new AbstractMap.SimpleEntry<>(SchemaElementType.MODEL, Arrays.asList(SchemaElementType.MODEL)), + new AbstractMap.SimpleEntry<>(SchemaElementType.VIEW, Arrays.asList(SchemaElementType.VIEW)), new AbstractMap.SimpleEntry<>(SchemaElementType.ID, Arrays.asList(SchemaElementType.ID)) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -51,12 +48,13 @@ public class ContextInheritParser implements SemanticParser { if (!shouldInherit(queryContext)) { return; } - ModelCluster modelCluster = getMatchedModelCluster(queryContext, chatContext); - if (modelCluster == null) { + Long viewId = getMatchedView(queryContext, chatContext); + if (viewId == null) { return; } - List elementMatches = queryContext.getModelClusterMapInfo() - .getMatchedElements(modelCluster.getKey()); + + List elementMatches = queryContext.getMapInfo().getMatchedElements(viewId); + List matchesToInherit = new ArrayList<>(); for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) { SchemaElementType matchType = match.getElement().getType(); @@ -72,17 +70,17 @@ public class ContextInheritParser implements SemanticParser { List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(queryContext, chatContext); - if (existSameQuery(query.getParseInfo().getModelClusterKey(), query.getQueryMode(), queryContext)) { + if (existSameQuery(query.getParseInfo().getViewId(), query.getQueryMode(), queryContext)) { continue; } queryContext.getCandidateQueries().add(query); } } - private boolean existSameQuery(String modelClusterKey, String queryMode, QueryContext queryContext) { + private boolean existSameQuery(Long viewId, String queryMode, QueryContext queryContext) { for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) { if (semanticQuery.getQueryMode().equals(queryMode) - && semanticQuery.getParseInfo().getModelClusterKey().equals(modelClusterKey)) { + && semanticQuery.getParseInfo().getViewId().equals(viewId)) { return true; } } @@ -111,25 +109,16 @@ public class ContextInheritParser implements SemanticParser { return metricModelQueries.size() == queryContext.getCandidateQueries().size(); } - protected ModelCluster getMatchedModelCluster(QueryContext queryContext, ChatContext chatContext) { - String contextModelClusterKey = chatContext.getParseInfo().getModelClusterKey(); - if (StringUtils.isBlank(contextModelClusterKey)) { + protected Long getMatchedView(QueryContext queryContext, ChatContext chatContext) { + Long viewId = chatContext.getParseInfo().getViewId(); + if (viewId == null) { return null; } - SemanticSchema semanticSchema = queryContext.getSemanticSchema(); - List allModelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema); - Set queryModelClusters = queryContext.getModelClusterMapInfo().getMatchedModelClusters(); - ModelCluster contextModelCluster = ModelCluster.build(contextModelClusterKey); - for (String cluster : queryModelClusters) { - ModelCluster queryModelCluster = ModelCluster.build(cluster); - for (ModelCluster modelCluster : allModelClusters) { - if (modelCluster.getModelIds().containsAll(contextModelCluster.getModelIds()) - && modelCluster.getModelIds().containsAll(queryModelCluster.getModelIds())) { - return queryModelCluster; - } - } + Set queryViews = queryContext.getMapInfo().getMatchedViewInfos(); + if (queryViews.contains(viewId)) { + return viewId; } - return null; + return viewId; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java index 291189b7f..916f25c0f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/RuleSqlParser.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.chat.core.parser.sql.rule; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.core.parser.SemanticParser; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; @@ -27,10 +27,10 @@ public class RuleSqlParser implements SemanticParser { @Override public void parse(QueryContext queryContext, ChatContext chatContext) { - SchemaModelClusterMapInfo modelClusterMapInfo = queryContext.getModelClusterMapInfo(); + SchemaMapInfo mapInfo = queryContext.getMapInfo(); // iterate all schemaElementMatches to resolve query mode - for (String modelClusterKey : modelClusterMapInfo.getMatchedModelClusters()) { - List elementMatches = modelClusterMapInfo.getMatchedElements(modelClusterKey); + for (Long viewId : mapInfo.getMatchedViewInfos()) { + List elementMatches = mapInfo.getMatchedElements(viewId); List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { query.fillParseInfo(queryContext, chatContext); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java index 11ffb1406..8f578986d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/Plugin.java @@ -20,7 +20,7 @@ public class Plugin extends RecordInfo { */ private String type; - private List modelList = Lists.newArrayList(); + private List viewList = Lists.newArrayList(); /** * description, for parsing @@ -52,7 +52,7 @@ public class Plugin extends RecordInfo { } public boolean isContainsAllModel() { - return CollectionUtils.isNotEmpty(modelList) && modelList.contains(-1L); + return CollectionUtils.isNotEmpty(viewList) && viewList.contains(-1L); } public Long getDefaultMode() { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java index d55d97f0f..a6963baa5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java @@ -23,6 +23,12 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,11 +38,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; @Slf4j @Component @@ -265,14 +266,14 @@ public class PluginManager { } private static Set getPluginMatchedModel(Plugin plugin, QueryContext queryContext) { - Set matchedModel = queryContext.getMapInfo().getMatchedModels(); + Set matchedViews = queryContext.getMapInfo().getMatchedViewInfos(); if (plugin.isContainsAllModel()) { return Sets.newHashSet(plugin.getDefaultMode()); } - List modelIds = plugin.getModelList(); + List modelIds = plugin.getViewList(); Set pluginMatchedModel = Sets.newHashSet(); for (Long modelId : modelIds) { - if (matchedModel.contains(modelId)) { + if (matchedViews.contains(modelId)) { pluginMatchedModel.add(modelId); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java index d4dd185ee..14d6225b1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginRecallResult.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; + import java.util.Set; @Data @@ -14,7 +15,7 @@ public class PluginRecallResult { private Plugin plugin; - private Set modelIds; + private Set viewIds; private double score; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java index 376961fcc..6c45f4b85 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.core.pojo; import com.fasterxml.jackson.annotation.JsonIgnore; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; @@ -12,15 +11,16 @@ import com.tencent.supersonic.chat.core.config.OptimizationConfig; import com.tencent.supersonic.chat.core.plugin.Plugin; import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; @Data @Builder @@ -30,14 +30,13 @@ public class QueryContext { private String queryText; private Integer chatId; - private Long modelId; + private Long viewId; private User user; private boolean saveAnswer = true; private Integer agentId; private QueryFilters queryFilters; private List candidateQueries = new ArrayList<>(); private SchemaMapInfo mapInfo = new SchemaMapInfo(); - private SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo(); @JsonIgnore private SemanticSchema semanticSchema; @JsonIgnore diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 7bd899a84..0a6942b69 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -19,15 +19,16 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; @Slf4j @ToString @@ -48,7 +49,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryType.SQL) .queryReq(QueryReqBuilder.buildS2SQLReq( - sqlInfo.getCorrectS2SQL(), parseInfo.getModel().getModelIds() + sqlInfo.getCorrectS2SQL(), parseInfo.getViewId() )) .build(); } else { @@ -83,7 +84,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } protected void convertBizNameToName(SemanticSchema semanticSchema, QueryStructReq queryStructReq) { - Map bizNameToName = semanticSchema.getBizNameToName(queryStructReq.getModelIdSet()); + Map bizNameToName = semanticSchema.getBizNameToName(queryStructReq.getViewId()); bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap()); List orders = queryStructReq.getOrders(); @@ -100,18 +101,17 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } List groups = queryStructReq.getGroups(); if (CollectionUtils.isNotEmpty(groups)) { - groups = groups.stream().map(group -> bizNameToName.get(group)).collect(Collectors.toList()); + groups = groups.stream().map(bizNameToName::get).collect(Collectors.toList()); queryStructReq.setGroups(groups); } List dimensionFilters = queryStructReq.getDimensionFilters(); if (CollectionUtils.isNotEmpty(dimensionFilters)) { - dimensionFilters.stream().forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); + dimensionFilters.forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } List metricFilters = queryStructReq.getMetricFilters(); if (CollectionUtils.isNotEmpty(dimensionFilters)) { - metricFilters.stream().forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); + metricFilters.forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } - queryStructReq.setModelName(parseInfo.getModelName()); } protected void initS2SqlByStruct(SemanticSchema semanticSchema) { @@ -121,9 +121,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } QueryStructReq queryStructReq = convertQueryStruct(); convertBizNameToName(semanticSchema, queryStructReq); - QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq); - parseInfo.getSqlInfo().setS2SQL(querySqlReq.getSql()); - parseInfo.getSqlInfo().setCorrectS2SQL(querySqlReq.getSql()); + QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); + parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java index effaf9d39..4745eb511 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMReq.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.chat.core.query.llm.s2sql; import com.fasterxml.jackson.annotation.JsonValue; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class LLMReq { @@ -35,7 +36,7 @@ public class LLMReq { private String domainName; - private String modelName; + private String viewName; private List fieldNameList; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java index da22871c4..b6a0b72e0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java @@ -42,8 +42,8 @@ public class LLMSqlQuery extends LLMSemanticQuery { long startTime = System.currentTimeMillis(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QuerySqlReq querySqlReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); - SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySqlReq, user); + QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getViewId()); + SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java index 491bb9327..78c54b009 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java @@ -79,7 +79,7 @@ public abstract class PluginSemanticQuery extends BaseSemanticQuery { if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) { for (ParamOption paramOption : webPage.getParamOptions()) { if (paramOption.getModelId() != null - && !parseInfo.getModel().getModelIds().contains(paramOption.getModelId())) { + && !parseInfo.getViewId().equals(paramOption.getModelId())) { continue; } paramOptions.add(paramOption); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java index 352f4cc2f..96e988878 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java @@ -3,14 +3,15 @@ package com.tencent.supersonic.chat.core.query.rule; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import lombok.Data; +import lombok.ToString; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.Data; -import lombok.ToString; @Data @ToString @@ -24,7 +25,7 @@ public class QueryMatcher { public QueryMatcher() { for (SchemaElementType type : SchemaElementType.values()) { - if (type.equals(SchemaElementType.MODEL)) { + if (type.equals(SchemaElementType.VIEW)) { elementOptionMap.put(type, QueryMatchOption.optional()); } else { elementOptionMap.put(type, QueryMatchOption.unused()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java index 0c8f76bb0..629450078 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java @@ -2,9 +2,6 @@ package com.tencent.supersonic.chat.core.query.rule; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; @@ -14,17 +11,19 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.core.config.OptimizationConfig; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.BaseSemanticQuery; +import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.chat.core.query.QueryManager; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -103,11 +102,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { } private void fillSchemaElement(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { - Set modelIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) - .map(SchemaElement::getModel).collect(Collectors.toSet()); - ModelCluster modelCluster = ModelCluster.build(modelIds); - modelCluster.buildName(semanticSchema.getModelIdToName()); - parseInfo.setModel(modelCluster); + Set viewIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) + .map(SchemaElement::getView).collect(Collectors.toSet()); + parseInfo.setView(semanticSchema.getView(viewIds.iterator().next())); Map> dim2Values = new HashMap<>(); Map> id2Values = new HashMap<>(); @@ -192,7 +189,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult execute(User user) { String queryMode = parseInfo.getQueryMode(); - if (StringUtils.isBlank(parseInfo.getModelClusterKey()) || StringUtils.isEmpty(queryMode) + if (parseInfo.getViewId() == null || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); @@ -233,7 +230,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult multiStructExecute(User user) { String queryMode = parseInfo.getQueryMode(); - if (StringUtils.isBlank(parseInfo.getModelClusterKey()) || StringUtils.isEmpty(queryMode) + if (parseInfo.getViewId() != null || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java index d05d45476..1845fad98 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java @@ -1,13 +1,12 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.MODEL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; - import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import org.springframework.stereotype.Component; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; @Component public class MetricModelQuery extends MetricSemanticQuery { @@ -15,7 +14,7 @@ public class MetricModelQuery extends MetricSemanticQuery { public MetricModelQuery() { super(); - queryMatcher.addOption(MODEL, OPTIONAL, AT_MOST, 1); + queryMatcher.addOption(SchemaElementType.VIEW, OPTIONAL, AT_MOST, 1); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java index 9e45e849d..74e07251d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java @@ -1,26 +1,11 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.core.config.AggregatorConfig; @@ -33,10 +18,15 @@ import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.RatioOverType; +import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; @@ -53,8 +43,19 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { @@ -75,30 +76,26 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { @Override public void fillParseInfo(QueryContext queryContext, ChatContext chatContext) { super.fillParseInfo(queryContext, chatContext); - parseInfo.setLimit(METRIC_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { - ChatConfigRichResp chatConfig = queryContext.getModelIdToChatRichConfig().get(parseInfo.getModelId()); - ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatAggRichConfig().getChatDefaultConfig(); + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); + TimeDefaultConfig timeDefaultConfig = viewSchema.getMetricTypeTimeDefaultConfig(); DateConf dateInfo = new DateConf(); - int unit = 1; - if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) { - unit = defaultConfig.getUnit(); + if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())) { + int unit = timeDefaultConfig.getUnit(); + String startDate = LocalDate.now().plusDays(-unit).toString(); + String endDate = startDate; + if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + } else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.RECENT); + endDate = LocalDate.now().plusDays(-1).toString(); + } + dateInfo.setUnit(unit); + dateInfo.setPeriod(timeDefaultConfig.getPeriod()); + dateInfo.setStartDate(startDate); + dateInfo.setEndDate(endDate); } - String startDate = LocalDate.now().plusDays(-unit).toString(); - String endDate = startDate; - - if (ChatDefaultConfigReq.TimeMode.LAST.equals(defaultConfig.getTimeMode())) { - dateInfo.setDateMode(DateConf.DateMode.BETWEEN); - } else if (ChatDefaultConfigReq.TimeMode.RECENT.equals(defaultConfig.getTimeMode())) { - dateInfo.setDateMode(DateConf.DateMode.RECENT); - endDate = LocalDate.now().plusDays(-1).toString(); - } - dateInfo.setUnit(unit); - dateInfo.setPeriod(defaultConfig.getPeriod()); - dateInfo.setStartDate(startDate); - dateInfo.setEndDate(endDate); - parseInfo.setDateInfo(dateInfo); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java index aa6494b39..3d20c29c7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java @@ -1,18 +1,20 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import org.apache.commons.collections.CollectionUtils; + import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; +import java.util.stream.Collectors; public abstract class TagListQuery extends TagSemanticQuery { @@ -23,28 +25,29 @@ public abstract class TagListQuery extends TagSemanticQuery { } private void addEntityDetailAndOrderByMetric(QueryContext queryContext, SemanticParseInfo parseInfo) { - Long modelId = parseInfo.getModelId(); - if (Objects.nonNull(modelId) && modelId > 0L) { - ChatConfigRichResp chaConfigRichDesc = queryContext.getModelIdToChatRichConfig().get(modelId); - ModelSchema modelSchema = queryContext.getSemanticSchema().getModelSchemaMap().get(parseInfo.getModelId()); - if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null - && Objects.nonNull(modelSchema) && Objects.nonNull(modelSchema.getEntity())) { + Long viewId = parseInfo.getViewId(); + if (Objects.nonNull(viewId) && viewId > 0L) { + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + if (viewSchema != null && Objects.nonNull(viewSchema.getEntity())) { Set dimensions = new LinkedHashSet<>(); - Set metrics = new LinkedHashSet(); - Set orders = new LinkedHashSet(); - ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc - .getChatDetailRichConfig().getChatDefaultConfig(); - if (chatDefaultConfig != null) { - if (CollectionUtils.isNotEmpty(chatDefaultConfig.getMetrics())) { - chatDefaultConfig.getMetrics().stream() - .forEach(metric -> { - metrics.add(metric); - orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)); - }); + Set metrics = new LinkedHashSet<>(); + Set orders = new LinkedHashSet<>(); + TagTypeDefaultConfig tagTypeDefaultConfig = viewSchema.getTagTypeDefaultConfig(); + if (tagTypeDefaultConfig != null) { + if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getMetricIds())) { + metrics = tagTypeDefaultConfig.getMetricIds() + .stream().map(id -> { + SchemaElement metric = viewSchema.getElement(SchemaElementType.METRIC, id); + if (metric != null) { + orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)); + } + return metric; + }).filter(Objects::nonNull).collect(Collectors.toSet()); } - if (CollectionUtils.isNotEmpty(chatDefaultConfig.getDimensions())) { - chatDefaultConfig.getDimensions().stream() - .forEach(dimension -> dimensions.add(dimension)); + if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDimensionIds())) { + dimensions = tagTypeDefaultConfig.getDimensionIds().stream() + .map(id -> viewSchema.getElement(SchemaElementType.DIMENSION, id)) + .filter(Objects::nonNull).collect(Collectors.toSet()); } } parseInfo.setDimensions(dimensions); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java index 706f6807e..c38a44184 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java @@ -1,21 +1,23 @@ package com.tencent.supersonic.chat.core.query.rule.tag; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.ChatContext; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import lombok.extern.slf4j.Slf4j; + import java.time.LocalDate; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j public abstract class TagSemanticQuery extends RuleSemanticQuery { @@ -40,19 +42,24 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery { parseInfo.setQueryType(QueryType.TAG); parseInfo.setLimit(TAG_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { - ChatConfigRichResp chatConfig = queryContext.getModelIdToChatRichConfig().get(parseInfo.getModelId()); - ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatDetailRichConfig().getChatDefaultConfig(); - - int unit = 1; - if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) { - unit = defaultConfig.getUnit(); - } - String date = LocalDate.now().plusDays(-unit).toString(); + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); + TimeDefaultConfig timeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.BETWEEN); - dateInfo.setStartDate(date); - dateInfo.setEndDate(date); - + if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())) { + int unit = timeDefaultConfig.getUnit(); + String startDate = LocalDate.now().plusDays(-unit).toString(); + String endDate = startDate; + if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + } else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.RECENT); + endDate = LocalDate.now().plusDays(-1).toString(); + } + dateInfo.setUnit(unit); + dateInfo.setPeriod(timeDefaultConfig.getPeriod()); + dateInfo.setStartDate(startDate); + dateInfo.setEndDate(endDate); + } parseInfo.setDateInfo(dateInfo); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java index c6e4e0ec1..0e47a8305 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.utils; import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.parser.JavaLLMProxy; import com.tencent.supersonic.chat.core.parser.LLMProxy; -import com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver; +import com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver; import com.tencent.supersonic.common.util.ContextUtils; import java.util.Map; import java.util.Objects; @@ -16,7 +16,7 @@ public class ComponentFactory { private static SemanticInterpreter semanticInterpreter; private static LLMProxy llmProxy; - private static ModelResolver modelResolver; + private static ViewResolver modelResolver; public static SemanticInterpreter getSemanticLayer() { if (Objects.isNull(semanticInterpreter)) { @@ -44,9 +44,9 @@ public class ComponentFactory { return llmProxy; } - public static ModelResolver getModelResolver() { + public static ViewResolver getModelResolver() { if (Objects.isNull(modelResolver)) { - modelResolver = init(ModelResolver.class); + modelResolver = init(ViewResolver.class); } return modelResolver; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java index ebc29fe91..21fe9be44 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java @@ -1,15 +1,9 @@ package com.tencent.supersonic.chat.core.utils; -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE; - import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.config.DefaultMetric; import com.tencent.supersonic.chat.core.config.Dim4Dict; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; @@ -18,8 +12,15 @@ import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -27,11 +28,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.StringJoiner; -import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE; @Slf4j @Component @@ -82,7 +84,7 @@ public class DictQueryHelper { if (!CollectionUtils.isEmpty(columns)) { for (QueryColumn column : columns) { - if (Strings.isNotEmpty(column.getNameEn())) { + if (StringUtils.isNotEmpty(column.getNameEn())) { String nameEn = column.getNameEn(); if (nameEn.endsWith(UNDERLINE_DOUBLE + bizName)) { dimNameRewrite = nameEn; @@ -159,9 +161,6 @@ public class DictQueryHelper { private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) { QueryStructReq queryStructCmd = new QueryStructReq(); - queryStructCmd.addModelId(modelId); - queryStructCmd.setGroups(Arrays.asList(dim4Dict.getBizName())); - List filters = generateFilters(dim4Dict, queryStructCmd); queryStructCmd.setDimensionFilters(filters); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ModelClusterBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ModelClusterBuilder.java deleted file mode 100644 index 853465e21..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ModelClusterBuilder.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.tencent.supersonic.chat.core.utils; - - -import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.common.pojo.ModelCluster; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -public class ModelClusterBuilder { - - public static List buildModelClusters(SemanticSchema semanticSchema) { - Map modelMap = semanticSchema.getModelSchemaMap(); - Set visited = new HashSet<>(); - List> modelClusters = new ArrayList<>(); - for (ModelSchema model : modelMap.values()) { - if (!visited.contains(model.getModel().getModel())) { - Set modelCluster = new HashSet<>(); - dfs(model, modelMap, visited, modelCluster); - modelClusters.add(modelCluster); - } - } - return modelClusters.stream().map(ModelCluster::build).collect(Collectors.toList()); - } - - private static void dfs(ModelSchema model, Map modelMap, - Set visited, Set modelCluster) { - visited.add(model.getModel().getModel()); - modelCluster.add(model.getModel().getModel()); - for (Long neighborId : model.getModelClusterSet()) { - if (!visited.contains(neighborId)) { - dfs(modelMap.get(neighborId), modelMap, visited, modelCluster); - } - } - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java index e877bbb8d..abd72eaf7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/NatureHelper.java @@ -3,8 +3,11 @@ package com.tencent.supersonic.chat.core.utils; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.core.knowledge.ModelInfoStat; +import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -12,8 +15,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; /** * nature parse helper @@ -37,8 +38,8 @@ public class NatureHelper { case ENTITY: result = SchemaElementType.ENTITY; break; - case MODEL: - result = SchemaElementType.MODEL; + case VIEW: + result = SchemaElementType.VIEW; break; case VALUE: result = SchemaElementType.VALUE; @@ -52,12 +53,12 @@ public class NatureHelper { return result; } - private static boolean isModelOrEntity(Term term, Integer model) { + private static boolean isViewOrEntity(Term term, Integer model) { return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString() .endsWith(DictWordType.ENTITY.getType()); } - public static Integer getModelByNature(Nature nature) { + public static Integer getViewByNature(Nature nature) { if (nature.startsWith(DictWordType.NATURE_SPILT)) { String[] dimensionValues = nature.toString().split(DictWordType.NATURE_SPILT); if (StringUtils.isNumeric(dimensionValues[1])) { @@ -67,7 +68,7 @@ public class NatureHelper { return 0; } - public static Long getModelId(String nature) { + public static Long getViewId(String nature) { try { String[] split = nature.split(DictWordType.NATURE_SPILT); if (split.length <= 1) { @@ -80,7 +81,7 @@ public class NatureHelper { return null; } - public static boolean isDimensionValueModelId(String nature) { + public static boolean isDimensionValueViewId(String nature) { if (StringUtils.isEmpty(nature)) { return false; } @@ -95,21 +96,21 @@ public class NatureHelper { && StringUtils.isNumeric(split[1]); } - public static ModelInfoStat getModelStat(List terms) { - return ModelInfoStat.builder() - .modelCount(getModelCount(terms)) - .dimensionModelCount(getDimensionCount(terms)) - .metricModelCount(getMetricCount(terms)) - .dimensionValueModelCount(getDimensionValueCount(terms)) + public static ViewInfoStat getViewStat(List terms) { + return ViewInfoStat.builder() + .viewCount(getViewCount(terms)) + .dimensionViewCount(getDimensionCount(terms)) + .metricViewCount(getMetricCount(terms)) + .dimensionValueViewCount(getDimensionValueCount(terms)) .build(); } - private static long getModelCount(List terms) { - return terms.stream().filter(term -> isModelOrEntity(term, getModelByNature(term.nature))).count(); + private static long getViewCount(List terms) { + return terms.stream().filter(term -> isViewOrEntity(term, getViewByNature(term.nature))).count(); } private static long getDimensionValueCount(List terms) { - return terms.stream().filter(term -> isDimensionValueModelId(term.nature.toString())).count(); + return terms.stream().filter(term -> isDimensionValueViewId(term.nature.toString())).count(); } private static long getDimensionCount(List terms) { @@ -129,13 +130,13 @@ public class NatureHelper { * @param terms * @return */ - public static Map> getModelToNatureStat(List terms) { + public static Map> getViewToNatureStat(List terms) { Map> modelToNature = new HashMap<>(); terms.stream().filter( term -> term.nature.startsWith(DictWordType.NATURE_SPILT) ).forEach(term -> { DictWordType dictWordType = DictWordType.getNatureType(String.valueOf(term.nature)); - Long model = getModelId(String.valueOf(term.nature)); + Long model = getViewId(String.valueOf(term.nature)); Map natureTypeMap = new HashMap<>(); natureTypeMap.put(dictWordType, 1); @@ -156,15 +157,15 @@ public class NatureHelper { return modelToNature; } - public static List selectPossibleModels(List terms) { - Map> modelToNatureStat = getModelToNatureStat(terms); - Integer maxModelTypeSize = modelToNatureStat.entrySet().stream() + public static List selectPossibleViews(List terms) { + Map> modelToNatureStat = getViewToNatureStat(terms); + Integer maxViewTypeSize = modelToNatureStat.entrySet().stream() .max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size()) .orElse(null); - if (Objects.isNull(maxModelTypeSize) || maxModelTypeSize == 0) { + if (Objects.isNull(maxViewTypeSize) || maxViewTypeSize == 0) { return new ArrayList<>(); } - return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxModelTypeSize) + return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxViewTypeSize) .map(entry -> entry.getKey()).collect(Collectors.toList()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java index 09ada33d4..5da9e4f2f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java @@ -36,39 +36,40 @@ import java.util.stream.Collectors; public class QueryReqBuilder { public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) { - QueryStructReq queryStructCmd = new QueryStructReq(); - queryStructCmd.setModelIds(parseInfo.getModel().getModelIds()); - queryStructCmd.setQueryType(parseInfo.getQueryType()); - queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); + QueryStructReq queryStructReq = new QueryStructReq(); + queryStructReq.setViewId(parseInfo.getViewId()); + queryStructReq.setViewName(parseInfo.getView().getName()); + queryStructReq.setQueryType(parseInfo.getQueryType()); + queryStructReq.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); List dimensionFilters = parseInfo.getDimensionFilters().stream() .filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName())) .map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue())) .collect(Collectors.toList()); - queryStructCmd.setDimensionFilters(dimensionFilters); + queryStructReq.setDimensionFilters(dimensionFilters); List metricFilters = parseInfo.getMetricFilters().stream() .map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue())) .collect(Collectors.toList()); - queryStructCmd.setMetricFilters(metricFilters); + queryStructReq.setMetricFilters(metricFilters); addDateDimension(parseInfo); List dimensions = parseInfo.getDimensions().stream().map(SchemaElement::getBizName) .collect(Collectors.toList()); - queryStructCmd.setGroups(dimensions); - queryStructCmd.setLimit(parseInfo.getLimit()); + queryStructReq.setGroups(dimensions); + queryStructReq.setLimit(parseInfo.getLimit()); // only one metric is queried at once Set metrics = parseInfo.getMetrics(); if (!CollectionUtils.isEmpty(metrics)) { SchemaElement metricElement = parseInfo.getMetrics().iterator().next(); Set order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), metricElement); - queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getAggType(), metricElement)); - queryStructCmd.setOrders(new ArrayList<>(order)); + queryStructReq.setAggregators(getAggregatorByMetric(parseInfo.getAggType(), metricElement)); + queryStructReq.setOrders(new ArrayList<>(order)); } - deletionDuplicated(queryStructCmd); + deletionDuplicated(queryStructReq); - return queryStructCmd; + return queryStructReq; } private static void deletionDuplicated(QueryStructReq queryStructReq) { @@ -118,7 +119,7 @@ public class QueryReqBuilder { for (Filter dimensionFilter : queryStructReq.getDimensionFilters()) { QueryStructReq req = new QueryStructReq(); BeanUtils.copyProperties(queryStructReq, req); - req.setModelIds(new HashSet<>(queryStructReq.getModelIds())); + req.setViewId(parseInfo.getViewId()); req.setDimensionFilters(Lists.newArrayList(dimensionFilter)); queryStructReqs.add(req); } @@ -130,16 +131,16 @@ public class QueryReqBuilder { * convert to QueryS2SQLReq * * @param querySql - * @param modelIds + * @param viewId * @return */ - public static QuerySqlReq buildS2SQLReq(String querySql, Set modelIds) { - QuerySqlReq querySqlReq = new QuerySqlReq(); + public static QuerySqlReq buildS2SQLReq(String querySql, Long viewId) { + QuerySqlReq querySQLReq = new QuerySqlReq(); if (Objects.nonNull(querySql)) { - querySqlReq.setSql(querySql); + querySQLReq.setSql(querySql); } - querySqlReq.setModelIds(modelIds); - return querySqlReq; + querySQLReq.setViewId(viewId); + return querySQLReq; } private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { @@ -234,14 +235,14 @@ public class QueryReqBuilder { public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum) { - QueryStructReq queryStructCmd = buildStructReq(parseInfo); - queryStructCmd.setQueryType(QueryType.METRIC); - queryStructCmd.setOrders(new ArrayList<>()); + QueryStructReq queryStructReq = buildStructReq(parseInfo); + queryStructReq.setQueryType(QueryType.METRIC); + queryStructReq.setOrders(new ArrayList<>()); List aggregators = new ArrayList<>(); Aggregator ratioRoll = new Aggregator(metric.getBizName(), aggOperatorEnum); aggregators.add(ratioRoll); - queryStructCmd.setAggregators(aggregators); - return queryStructCmd; + queryStructReq.setAggregators(aggregators); + return queryStructReq; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java index 94e8f1d78..8eacf4246 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SimilarQueryManager.java @@ -57,7 +57,7 @@ public class SimilarQueryManager { embeddingQuery.setQuery(queryText); Map metaData = new HashMap<>(); - metaData.put("modelId", (similarQueryReq.getModelId())); + metaData.put("modelId", similarQueryReq.getViewId()); metaData.put("agentId", similarQueryReq.getAgentId()); embeddingQuery.setMetadata(metaData); String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection(); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java index 5df18b5dc..eb173d54d 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java @@ -1,15 +1,16 @@ package com.tencent.supersonic.chat.core.s2sql; -import static org.mockito.Mockito.when; - import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; + +import static org.mockito.Mockito.when; class LLMSqlParserTest { @@ -28,7 +29,7 @@ class LLMSqlParserTest { SchemaElement schemaElement = SchemaElement.builder() .bizName("singer_name") .name("歌手名") - .model(2L) + .view(2L) .schemaValueMaps(schemaValueMaps) .build(); dimensions.add(schemaElement); @@ -36,7 +37,7 @@ class LLMSqlParserTest { SchemaElement schemaElement2 = SchemaElement.builder() .bizName("publish_time") .name("发布时间") - .model(2L) + .view(2L) .build(); dimensions.add(schemaElement2); @@ -46,7 +47,7 @@ class LLMSqlParserTest { SchemaElement metric = SchemaElement.builder() .bizName("play_count") .name("播放量") - .model(2L) + .view(2L) .build(); metrics.add(metric); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java index 682aa9c62..5707e3ea0 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java @@ -32,8 +32,8 @@ public class SchemaDictUpdateListener implements ApplicationListener DictWord dictWord = new DictWord(); dictWord.setWord(dataItem.getName()); String sign = DictWordType.NATURE_SPILT; - String nature = sign + dataItem.getModelId() + sign + dataItem.getId() - + sign + dataItem.getType().getName(); + String nature = sign + 1 + sign + dataItem.getId() + + sign + dataItem.getType().name().toLowerCase(); String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; dictWord.setNature(nature); dictWord.setNatureWithFrequency(natureWithFrequency); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java index 8e5d64ddf..3d6a18d22 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/PluginDO.java @@ -1,256 +1,40 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; import java.util.Date; +@Data +@TableName("s2_plugin") public class PluginDO { - /** - * - */ + + @TableId(type = IdType.AUTO) private Long id; - /** - * DASHBOARD,WIDGET,URL - */ private String type; - /** - * - */ - private String model; + private String view; - /** - * - */ private String pattern; - /** - * - */ private String parseMode; - /** - * - */ private String name; - /** - * - */ private Date createdAt; - /** - * - */ private String createdBy; - /** - * - */ private Date updatedAt; - /** - * - */ private String updatedBy; - /** - * - */ private String parseModeConfig; - /** - * - */ private String config; - /** - * - */ private String comment; - /** - * @return id - */ - public Long getId() { - return id; - } - - /** - * @param id - */ - public void setId(Long id) { - this.id = id; - } - - /** - * DASHBOARD,WIDGET,URL - * - * @return type DASHBOARD,WIDGET,URL - */ - public String getType() { - return type; - } - - /** - * DASHBOARD,WIDGET,URL - * - * @param type DASHBOARD,WIDGET,URL - */ - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - /** - * @return model - */ - public String getModel() { - return model; - } - - /** - * @param model - */ - public void setModel(String model) { - this.model = model == null ? null : model.trim(); - } - - /** - * @return pattern - */ - public String getPattern() { - return pattern; - } - - /** - * @param pattern - */ - public void setPattern(String pattern) { - this.pattern = pattern == null ? null : pattern.trim(); - } - - /** - * @return parse_mode - */ - public String getParseMode() { - return parseMode; - } - - /** - * @param parseMode - */ - public void setParseMode(String parseMode) { - this.parseMode = parseMode == null ? null : parseMode.trim(); - } - - /** - * @return name - */ - public String getName() { - return name; - } - - /** - * @param name - */ - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - /** - * @return created_at - */ - public Date getCreatedAt() { - return createdAt; - } - - /** - * @param createdAt - */ - public void setCreatedAt(Date createdAt) { - this.createdAt = createdAt; - } - - /** - * @return created_by - */ - public String getCreatedBy() { - return createdBy; - } - - /** - * @param createdBy - */ - public void setCreatedBy(String createdBy) { - this.createdBy = createdBy == null ? null : createdBy.trim(); - } - - /** - * @return updated_at - */ - public Date getUpdatedAt() { - return updatedAt; - } - - /** - * @param updatedAt - */ - public void setUpdatedAt(Date updatedAt) { - this.updatedAt = updatedAt; - } - - /** - * @return updated_by - */ - public String getUpdatedBy() { - return updatedBy; - } - - /** - * @param updatedBy - */ - public void setUpdatedBy(String updatedBy) { - this.updatedBy = updatedBy == null ? null : updatedBy.trim(); - } - - /** - * @return parse_mode_config - */ - public String getParseModeConfig() { - return parseModeConfig; - } - - /** - * @param parseModeConfig - */ - public void setParseModeConfig(String parseModeConfig) { - this.parseModeConfig = parseModeConfig == null ? null : parseModeConfig.trim(); - } - - /** - * @return config - */ - public String getConfig() { - return config; - } - - /** - * @param config - */ - public void setConfig(String config) { - this.config = config == null ? null : config.trim(); - } - - /** - * @return comment - */ - public String getComment() { - return comment; - } - - /** - * @param comment - */ - public void setComment(String comment) { - this.comment = comment == null ? null : comment.trim(); - } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java index c0856c18b..3f0769904 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/PluginDOMapper.java @@ -1,69 +1,10 @@ package com.tencent.supersonic.chat.server.persistence.mapper; -import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; import org.apache.ibatis.annotations.Mapper; -import java.util.List; @Mapper -public interface PluginDOMapper { - /** - * - * @mbg.generated - */ - long countByExample(PluginDOExample example); +public interface PluginDOMapper extends BaseMapper { - /** - * - * @mbg.generated - */ - int deleteByPrimaryKey(Long id); - - /** - * - * @mbg.generated - */ - int insert(PluginDO record); - - /** - * - * @mbg.generated - */ - int insertSelective(PluginDO record); - - /** - * - * @mbg.generated - */ - List selectByExampleWithBLOBs(PluginDOExample example); - - /** - * - * @mbg.generated - */ - List selectByExample(PluginDOExample example); - - /** - * - * @mbg.generated - */ - PluginDO selectByPrimaryKey(Long id); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKeySelective(PluginDO record); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKeyWithBLOBs(PluginDO record); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKey(PluginDO record); } \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/PluginRepository.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/PluginRepository.java index c24e50910..66ed38a28 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/PluginRepository.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/PluginRepository.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.server.persistence.repository; -import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; import java.util.List; @@ -16,7 +16,7 @@ public interface PluginRepository { PluginDO getPlugin(Long id); - List query(PluginDOExample pluginDOExample); + List query(QueryWrapper queryWrapper); void deletePlugin(Long id); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java index 47cc87165..058272529 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/PluginRepositoryImpl.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.server.persistence.repository.impl; -import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; import com.tencent.supersonic.chat.server.persistence.mapper.PluginDOMapper; import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository; @@ -26,7 +26,7 @@ public class PluginRepositoryImpl implements PluginRepository { @Override public List getPlugins() { - return pluginDOMapper.selectByExampleWithBLOBs(new PluginDOExample()); + return pluginDOMapper.selectList(new QueryWrapper<>()); } @Override @@ -60,22 +60,22 @@ public class PluginRepositoryImpl implements PluginRepository { @Override public void updatePlugin(PluginDO pluginDO) { - pluginDOMapper.updateByPrimaryKeyWithBLOBs(pluginDO); + pluginDOMapper.updateById(pluginDO); } @Override public PluginDO getPlugin(Long id) { - return pluginDOMapper.selectByPrimaryKey(id); + return pluginDOMapper.selectById(id); } @Override - public List query(PluginDOExample pluginDOExample) { - return pluginDOMapper.selectByExampleWithBLOBs(pluginDOExample); + public List query(QueryWrapper queryWrapper) { + return pluginDOMapper.selectList(queryWrapper); } @Override public void deletePlugin(Long id) { - pluginDOMapper.deleteByPrimaryKey(id); + pluginDOMapper.deleteById(id); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index 056305039..dda9c80d3 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -1,22 +1,23 @@ package com.tencent.supersonic.chat.server.processor.execute; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.server.service.SemanticService; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; +import org.springframework.util.CollectionUtils; + import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; /** * DimensionRecommendProcessor recommend some dimensions @@ -33,13 +34,13 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { return; } SchemaElement element = semanticParseInfo.getMetrics().iterator().next(); - List dimensionRecommended = getDimensions(element.getId(), element.getModel()); + List dimensionRecommended = getDimensions(element.getId(), element.getView()); queryResult.setRecommendedDimensions(dimensionRecommended); } private List getDimensions(Long metricId, Long modelId) { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema modelSchema = semanticService.getModelSchema(modelId); List drillDownDimensions = Lists.newArrayList(); Set metricElements = modelSchema.getMetrics(); if (!CollectionUtils.isEmpty(metricElements)) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java index 634aae1a3..c91f4deff 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java @@ -14,6 +14,8 @@ import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQuery; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; +import org.springframework.util.CollectionUtils; + import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -21,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; /** * MetricRecommendProcessor fills recommended metrics based on embedding similarity. @@ -45,7 +46,7 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { } List metricNames = Collections.singletonList(parseInfo.getMetrics().iterator().next().getName()); Map filterCondition = new HashMap<>(); - filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getModel().toString()); + filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getView().toString()); filterCondition.put("type", SchemaElementType.METRIC.name()); RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames) .filterCondition(filterCondition).queryEmbeddings(null).build(); @@ -70,9 +71,9 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor { if (!metricIds.contains(Retrieval.getLongId(retrieval.getId()))) { SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(retrieval.getMetadata()), SchemaElement.class); - if (retrieval.getMetadata().containsKey("modelId")) { - String modelId = retrieval.getMetadata().get("modelId").toString(); - schemaElement.setModel(Long.parseLong(modelId)); + if (retrieval.getMetadata().containsKey("viewId")) { + String viewId = retrieval.getMetadata().get("viewId").toString(); + schemaElement.setView(Long.parseLong(viewId)); } schemaElement.setOrder(++metricOrder); parseInfo.getMetrics().add(schemaElement); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java index 5d018ffec..30e6c4eb0 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.server.processor.parse; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.core.pojo.ChatContext; @@ -10,9 +11,10 @@ import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.core.query.llm.analytics.MetricAnalyzeQuery; import com.tencent.supersonic.chat.server.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; +import org.springframework.util.CollectionUtils; + import java.util.List; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; /** * EntityInfoProcessor fills core attributes of an entity so that @@ -35,8 +37,9 @@ public class EntityInfoProcessor implements ParseResultProcessor { return; } //1. set entity info + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryContext.getUser()); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, queryContext.getUser()); if (QueryManager.isTagQuery(queryMode) || QueryManager.isMetricQuery(queryMode)) { parseInfo.setEntityInfo(entityInfo); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java index 9d948ad97..13b37cd23 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java @@ -1,15 +1,15 @@ package com.tencent.supersonic.chat.server.processor.parse; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; +import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.chat.core.query.SemanticQuery; import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; @@ -18,6 +18,11 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -26,10 +31,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.util.CollectionUtils; /** * ParseInfoProcessor extracts structured info from S2SQL so that @@ -73,9 +74,9 @@ public class ParseInfoProcessor implements ParseResultProcessor { } //set filter - Set modelIds = parseInfo.getModel().getModelIds(); + Long viewId = parseInfo.getViewId(); try { - Map fieldNameToElement = getNameToElement(modelIds); + Map fieldNameToElement = getNameToElement(viewId); List result = getDimensionFilter(fieldNameToElement, expressions); parseInfo.getDimensionFilters().addAll(result); } catch (Exception e) { @@ -87,31 +88,31 @@ public class ParseInfoProcessor implements ParseResultProcessor { return; } List allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); - Set metrics = getElements(modelIds, allFields, semanticSchema.getMetrics()); + Set metrics = getElements(viewId, allFields, semanticSchema.getMetrics()); parseInfo.setMetrics(metrics); if (QueryType.METRIC.equals(parseInfo.getQueryType())) { List groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); List groupByDimensions = getFieldsExceptDate(groupByFields); - parseInfo.setDimensions(getElements(modelIds, groupByDimensions, semanticSchema.getDimensions())); + parseInfo.setDimensions(getElements(viewId, groupByDimensions, semanticSchema.getDimensions())); } else if (QueryType.TAG.equals(parseInfo.getQueryType())) { List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); List selectDimensions = getFieldsExceptDate(selectFields); - parseInfo.setDimensions(getElements(modelIds, selectDimensions, semanticSchema.getDimensions())); + parseInfo.setDimensions(getElements(viewId, selectDimensions, semanticSchema.getDimensions())); } } - private Set getElements(Set modelIds, List allFields, List elements) { + private Set getElements(Long viewId, List allFields, List elements) { return elements.stream() .filter(schemaElement -> { if (CollectionUtils.isEmpty(schemaElement.getAlias())) { - return modelIds.contains(schemaElement.getModel()) && allFields.contains( + return viewId.equals(schemaElement.getView()) && allFields.contains( schemaElement.getName()); } Set allFieldsSet = new HashSet<>(allFields); Set aliasSet = new HashSet<>(schemaElement.getAlias()); List intersection = allFieldsSet.stream() .filter(aliasSet::contains).collect(Collectors.toList()); - return modelIds.contains(schemaElement.getModel()) && (allFields.contains( + return viewId.equals(schemaElement.getView()) && (allFields.contains( schemaElement.getName()) || !CollectionUtils.isEmpty(intersection)); } ).collect(Collectors.toSet()); @@ -193,10 +194,10 @@ public class ParseInfoProcessor implements ParseResultProcessor { return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); } - protected Map getNameToElement(Set modelIds) { + protected Map getNameToElement(Long viewId) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - List dimensions = semanticSchema.getDimensions(modelIds); - List metrics = semanticSchema.getMetrics(modelIds); + List dimensions = semanticSchema.getDimensions(viewId); + List metrics = semanticSchema.getMetrics(viewId); List allElements = Lists.newArrayList(); allElements.addAll(dimensions); @@ -218,4 +219,4 @@ public class ParseInfoProcessor implements ParseResultProcessor { (value1, value2) -> value2)); } -} +} \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java index 140c7d9e0..125b71f10 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java @@ -3,21 +3,20 @@ package com.tencent.supersonic.chat.server.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.server.service.ConfigService; -import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -76,21 +75,6 @@ public class ChatConfigController { return configService.getAllChatRichConfig(); } - @GetMapping("/modelList/{domainId}") - public List getModelList(@PathVariable("domainId") Long domainId, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return semanticInterpreter.getModelList(AuthType.ADMIN, domainId, user); - } - - @GetMapping("/modelList") - public List getModelList(HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return semanticInterpreter.getModelList(AuthType.ADMIN, null, user); - } - @GetMapping("/domainList") public List getDomainList(HttpServletRequest request, HttpServletResponse response) { @@ -98,17 +82,13 @@ public class ChatConfigController { return semanticInterpreter.getDomainList(user); } - @GetMapping("/modelList/view") - public List getModelListVisible(HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return semanticInterpreter.getModelList(AuthType.VISIBLE, null, user); + @GetMapping("/viewList/{domainId}") + public List getViewList(@PathVariable("domainId") Long domainId) { + return semanticInterpreter.getViewList(domainId); } @PostMapping("/dimension/page") - public PageInfo getDimension(@RequestBody PageDimensionReq pageDimensionReq, - HttpServletRequest request, - HttpServletResponse response) { + public PageInfo getDimension(@RequestBody PageDimensionReq pageDimensionReq) { return semanticInterpreter.getDimensionPage(pageDimensionReq); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java index c45c9a8c5..c7e74cce3 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java @@ -13,7 +13,6 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO; import java.util.List; -import java.util.Set; public interface ChatService { @@ -22,7 +21,7 @@ public interface ChatService { * @param chatId * @return */ - Set getContextModel(Integer chatId); + Long getContextModel(Integer chatId); ChatContext getOrCreateContext(int chatId); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index 4cbbb4d17..235fa53ed 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -1,41 +1,27 @@ package com.tencent.supersonic.chat.server.service; -import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.DataInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; -import com.tencent.supersonic.chat.api.pojo.response.ModelInfo; -import com.tencent.supersonic.chat.core.config.AggregatorConfig; +import com.tencent.supersonic.chat.api.pojo.response.ViewInfo; import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.chat.server.service.impl.SchemaService; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.DateConf.DateMode; -import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -43,16 +29,20 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.time.LocalDate; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Service @Slf4j public class SemanticService { @Autowired private SchemaService schemaService; - @Autowired - private ConfigService configService; - @Autowired - private AggregatorConfig aggregatorConfig; private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); @@ -60,31 +50,32 @@ public class SemanticService { return schemaService.getSemanticSchema(); } - public ModelSchema getModelSchema(Long id) { - return schemaService.getModelSchema(id); + public ViewSchema getModelSchema(Long id) { + return schemaService.getViewSchema(id); } - public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) { - if (parseInfo != null && parseInfo.getModelId() > 0) { - EntityInfo entityInfo = getEntityInfo(parseInfo.getModelId()); - if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getModelInfo() == null) { + public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, ViewSchema viewSchema, User user) { + if (parseInfo != null && parseInfo.getViewId() > 0) { + EntityInfo entityInfo = getEntityBasicInfo(viewSchema); + if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getViewInfo() == null) { entityInfo.setMetrics(null); entityInfo.setDimensions(null); return entityInfo; } - String primaryKey = entityInfo.getModelInfo().getPrimaryKey(); + String primaryKey = entityInfo.getViewInfo().getPrimaryKey(); if (StringUtils.isNotBlank(primaryKey)) { - String modelInfoId = ""; + String entityId = ""; for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName() .equals(primaryKey)) { if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { - modelInfoId = chatFilter.getValue().toString(); + entityId = chatFilter.getValue().toString(); } } } + entityInfo.setEntityId(entityId); try { - setMainModel(entityInfo, parseInfo, modelInfoId, user); + fillEntityInfoValue(entityInfo, viewSchema, user); return entityInfo; } catch (Exception e) { log.error("setMainModel error", e); @@ -94,77 +85,50 @@ public class SemanticService { return null; } - public EntityInfo getEntityInfo(Long model) { - ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(model); - if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) { - return new EntityInfo(); - } - return getEntityInfo(chaConfigRichDesc); - } - - private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) { + private EntityInfo getEntityBasicInfo(ViewSchema viewSchema) { EntityInfo entityInfo = new EntityInfo(); - Long modelId = chaConfigRichDesc.getModelId(); - if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(modelId)) { - SemanticService schemaService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = schemaService.getModelSchema(modelId); - if (Objects.isNull(modelSchema) || Objects.isNull(modelSchema.getEntity())) { - return entityInfo; - } - ModelInfo modelInfo = new ModelInfo(); - modelInfo.setItemId(modelId.intValue()); - modelInfo.setName(modelSchema.getModel().getName()); - modelInfo.setWords(modelSchema.getModel().getAlias()); - modelInfo.setBizName(modelSchema.getModel().getBizName()); - if (Objects.nonNull(modelSchema.getEntity())) { - modelInfo.setPrimaryKey(modelSchema.getEntity().getBizName()); - } - - entityInfo.setModelInfo(modelInfo); - List dimensions = new ArrayList<>(); - List metrics = new ArrayList<>(); - - if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig()) - && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) { - ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig() - .getChatDefaultConfig(); - if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())) { - for (SchemaElement dimensionDesc : chatDefaultConfig.getDimensions()) { - DataInfo mainEntityDimension = new DataInfo(); - mainEntityDimension.setItemId(dimensionDesc.getId().intValue()); - mainEntityDimension.setName(dimensionDesc.getName()); - mainEntityDimension.setBizName(dimensionDesc.getBizName()); - dimensions.add(mainEntityDimension); - } - entityInfo.setDimensions(dimensions); - } - - if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) { - for (SchemaElement metricDesc : chatDefaultConfig.getMetrics()) { - DataInfo dataInfo = new DataInfo(); - dataInfo.setName(metricDesc.getName()); - dataInfo.setBizName(metricDesc.getBizName()); - dataInfo.setItemId(metricDesc.getId().intValue()); - metrics.add(dataInfo); - } - entityInfo.setMetrics(metrics); - } - } + if (viewSchema == null) { + return entityInfo; } + Long viewId = viewSchema.getView().getView(); + ViewInfo viewInfo = new ViewInfo(); + viewInfo.setItemId(viewId.intValue()); + viewInfo.setName(viewSchema.getView().getName()); + viewInfo.setWords(viewSchema.getView().getAlias()); + viewInfo.setBizName(viewSchema.getView().getBizName()); + if (Objects.nonNull(viewSchema.getEntity())) { + viewInfo.setPrimaryKey(viewSchema.getEntity().getBizName()); + } + entityInfo.setViewInfo(viewInfo); + TagTypeDefaultConfig tagTypeDefaultConfig = viewSchema.getTagTypeDefaultConfig(); + if (tagTypeDefaultConfig == null) { + return entityInfo; + } + List dimensions = tagTypeDefaultConfig.getDimensionIds().stream() + .map(id -> { + SchemaElement element = viewSchema.getElement(SchemaElementType.DIMENSION, id); + if (element == null) { + return null; + } + return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); + }).filter(Objects::nonNull).collect(Collectors.toList()); + List metrics = tagTypeDefaultConfig.getDimensionIds().stream() + .map(id -> { + SchemaElement element = viewSchema.getElement(SchemaElementType.METRIC, id); + if (element == null) { + return null; + } + return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); + }).filter(Objects::nonNull).collect(Collectors.toList()); + entityInfo.setDimensions(dimensions); + entityInfo.setMetrics(metrics); return entityInfo; } - public void setMainModel(EntityInfo modelInfo, SemanticParseInfo parseInfo, String entity, User user) { - if (StringUtils.isEmpty(entity)) { - return; - } - - List entities = Collections.singletonList(entity); - - SemanticQueryResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities, - user); - + public void fillEntityInfoValue(EntityInfo entityInfo, ViewSchema viewSchema, User user) { + SemanticQueryResp queryResultWithColumns = + getQueryResultWithSchemaResp(entityInfo, viewSchema, user); if (queryResultWithColumns != null) { if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList()) && queryResultWithColumns.getResultList().size() > 0) { @@ -174,47 +138,38 @@ public class SemanticService { if (entry.getValue() == null || entryKey == null) { continue; } - modelInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) + entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) .forEach(i -> i.setValue(entry.getValue().toString())); - modelInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) + entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) .forEach(i -> i.setValue(entry.getValue().toString())); } } } } - public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo, - List entities, User user) { - if (CollectionUtils.isEmpty(entities)) { - return null; - } - ModelSchema modelSchema = schemaService.getModelSchema(parseInfo.getModelId()); - modelInfo.setEntityId(entities.get(0)); + public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, ViewSchema viewSchema, User user) { SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(parseInfo.getModelId()))); + semanticParseInfo.setView(viewSchema.getView()); semanticParseInfo.setQueryType(QueryType.TAG); - semanticParseInfo.setMetrics(getMetrics(modelInfo)); - semanticParseInfo.setDimensions(getDimensions(modelInfo)); + semanticParseInfo.setMetrics(getMetrics(entityInfo)); + semanticParseInfo.setDimensions(getDimensions(entityInfo)); DateConf dateInfo = new DateConf(); int unit = 1; - ChatConfigResp chatConfigInfo = - configService.fetchConfigByModelId(modelSchema.getModel().getId()); - if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(chatConfigInfo.getChatDetailConfig()) - && Objects.nonNull(chatConfigInfo.getChatDetailConfig().getChatDefaultConfig())) { - ChatDefaultConfigReq chatDefaultConfig = chatConfigInfo.getChatDetailConfig().getChatDefaultConfig(); - unit = chatDefaultConfig.getUnit(); + TimeDefaultConfig timeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + if (Objects.nonNull(timeDefaultConfig)) { + unit = timeDefaultConfig.getUnit(); String date = LocalDate.now().plusDays(-unit).toString(); - dateInfo.setDateMode(DateMode.BETWEEN); + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); dateInfo.setStartDate(date); dateInfo.setEndDate(date); } else { dateInfo.setUnit(unit); - dateInfo.setDateMode(DateMode.RECENT); + dateInfo.setDateMode(DateConf.DateMode.RECENT); } semanticParseInfo.setDateInfo(dateInfo); // add filter - QueryFilter chatFilter = getQueryFilter(modelInfo, entities); + QueryFilter chatFilter = getQueryFilter(entityInfo); Set chatFilters = new LinkedHashSet(); chatFilters.add(chatFilter); semanticParseInfo.setDimensionFilters(chatFilters); @@ -229,16 +184,11 @@ public class SemanticService { return queryResultWithColumns; } - private QueryFilter getQueryFilter(EntityInfo modelInfo, List entities) { + private QueryFilter getQueryFilter(EntityInfo entityInfo) { QueryFilter chatFilter = new QueryFilter(); - if (entities.size() == 1) { - chatFilter.setValue(entities.get(0)); - chatFilter.setOperator(FilterOperatorEnum.EQUALS); - } else { - chatFilter.setValue(entities); - chatFilter.setOperator(FilterOperatorEnum.IN); - } - chatFilter.setBizName(getEntityPrimaryName(modelInfo)); + chatFilter.setValue(entityInfo.getEntityId()); + chatFilter.setOperator(FilterOperatorEnum.EQUALS); + chatFilter.setBizName(getEntityPrimaryName(entityInfo)); return chatFilter; } @@ -271,8 +221,8 @@ public class SemanticService { return metrics; } - private String getEntityPrimaryName(EntityInfo modelInfo) { - return modelInfo.getModelInfo().getPrimaryKey(); + private String getEntityPrimaryName(EntityInfo entityInfo) { + return entityInfo.getViewInfo().getPrimaryKey(); } } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index e21960bdd..2b8f365b9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -31,7 +31,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -52,7 +51,7 @@ public class ChatServiceImpl implements ChatService { } @Override - public Set getContextModel(Integer chatId) { + public Long getContextModel(Integer chatId) { if (Objects.isNull(chatId)) { return null; } @@ -61,8 +60,8 @@ public class ChatServiceImpl implements ChatService { return null; } SemanticParseInfo originalSemanticParse = chatContext.getParseInfo(); - if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getModel().getModelIds())) { - return originalSemanticParse.getModel().getModelIds(); + if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getViewId())) { + return originalSemanticParse.getViewId(); } return null; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index ca17ee1d7..02f430678 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -173,7 +173,7 @@ public class ConfigServiceImpl implements ConfigService { } private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility, - ModelSchema modelSchema) { + ViewSchema modelSchema) { ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo(); List dimIdAllList = chatConfigHelper.generateAllDimIdList(modelSchema); @@ -215,22 +215,22 @@ public class ConfigServiceImpl implements ConfigService { } BeanUtils.copyProperties(chatConfigResp, chatConfigRich); - ModelSchema modelSchema = semanticService.getModelSchema(modelId); - if (modelSchema == null) { + ViewSchema viewSchema = semanticService.getModelSchema(modelId); + if (viewSchema == null) { return chatConfigRich; } - chatConfigRich.setBizName(modelSchema.getModel().getBizName()); - chatConfigRich.setModelName(modelSchema.getModel().getName()); + chatConfigRich.setBizName(viewSchema.getView().getBizName()); + chatConfigRich.setModelName(viewSchema.getView().getName()); - chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(modelSchema, chatConfigResp)); - chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(modelSchema, chatConfigRich, chatConfigResp)); + chatConfigRich.setChatAggRichConfig(fillChatAggRichConfig(viewSchema, chatConfigResp)); + chatConfigRich.setChatDetailRichConfig(fillChatDetailRichConfig(viewSchema, chatConfigRich, chatConfigResp)); return chatConfigRich; } - private ChatDetailRichConfigResp fillChatDetailRichConfig(ModelSchema modelSchema, - ChatConfigRichResp chatConfigRich, - ChatConfigResp chatConfigResp) { + private ChatDetailRichConfigResp fillChatDetailRichConfig(ViewSchema modelSchema, + ChatConfigRichResp chatConfigRich, + ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) { return null; } @@ -247,7 +247,7 @@ public class ConfigServiceImpl implements ConfigService { return detailRichConfig; } - private EntityRichInfoResp generateRichEntity(Entity entity, ModelSchema modelSchema) { + private EntityRichInfoResp generateRichEntity(Entity entity, ViewSchema modelSchema) { EntityRichInfoResp entityRichInfo = new EntityRichInfoResp(); if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) { return entityRichInfo; @@ -260,7 +260,7 @@ public class ConfigServiceImpl implements ConfigService { return entityRichInfo; } - private ChatAggRichConfigResp fillChatAggRichConfig(ModelSchema modelSchema, ChatConfigResp chatConfigResp) { + private ChatAggRichConfigResp fillChatAggRichConfig(ViewSchema modelSchema, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) { return null; } @@ -277,7 +277,7 @@ public class ConfigServiceImpl implements ConfigService { } private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig, - ModelSchema modelSchema, + ViewSchema modelSchema, ItemVisibilityInfo itemVisibilityInfo) { ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp(); if (Objects.isNull(chatDefaultConfig)) { @@ -327,7 +327,7 @@ public class ConfigServiceImpl implements ConfigService { } private List fillKnowledgeBizName(List knowledgeInfos, - ModelSchema modelSchema) { + ViewSchema modelSchema) { if (CollectionUtils.isEmpty(knowledgeInfos)) { return new ArrayList<>(); } @@ -347,9 +347,9 @@ public class ConfigServiceImpl implements ConfigService { @Override public List getAllChatRichConfig() { List chatConfigRichInfoList = new ArrayList<>(); - List modelSchemas = semanticInterpreter.getModelSchema(); + List modelSchemas = semanticInterpreter.getViewSchema(); modelSchemas.stream().forEach(modelSchema -> { - ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getModel().getId()); + ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(modelSchema.getView().getId()); if (Objects.nonNull(chatConfigRichInfo)) { chatConfigRichInfoList.add(chatConfigRichInfo); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java index 77a82182f..f0dff6895 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/PluginServiceImpl.java @@ -1,22 +1,25 @@ package com.tencent.supersonic.chat.server.service.impl; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.plugin.Plugin; import com.tencent.supersonic.chat.core.plugin.PluginParseConfig; import com.tencent.supersonic.chat.core.plugin.event.PluginAddEvent; import com.tencent.supersonic.chat.core.plugin.event.PluginDelEvent; import com.tencent.supersonic.chat.core.plugin.event.PluginUpdateEvent; -import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO; -import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDOExample; import com.tencent.supersonic.chat.server.persistence.repository.PluginRepository; import com.tencent.supersonic.chat.server.service.PluginService; -import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.Arrays; import java.util.Date; import java.util.List; @@ -24,12 +27,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -91,27 +88,27 @@ public class PluginServiceImpl implements PluginService { @Override public List query(PluginQueryReq pluginQueryReq) { - PluginDOExample pluginDOExample = new PluginDOExample(); - pluginDOExample.createCriteria(); + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (StringUtils.isNotBlank(pluginQueryReq.getType())) { - pluginDOExample.getOredCriteria().get(0).andTypeEqualTo(pluginQueryReq.getType()); + queryWrapper.lambda().eq(PluginDO::getType, pluginQueryReq.getType()); } - if (StringUtils.isNotBlank(pluginQueryReq.getModel())) { - pluginDOExample.getOredCriteria().get(0).andModelLike('%' + pluginQueryReq.getModel() + '%'); + if (StringUtils.isNotBlank(pluginQueryReq.getView())) { + queryWrapper.lambda().like(PluginDO::getView, pluginQueryReq.getView()); } if (StringUtils.isNotBlank(pluginQueryReq.getParseMode())) { - pluginDOExample.getOredCriteria().get(0).andParseModeEqualTo(pluginQueryReq.getParseMode()); + queryWrapper.lambda().eq(PluginDO::getParseMode, pluginQueryReq.getParseMode()); } if (StringUtils.isNotBlank(pluginQueryReq.getName())) { - pluginDOExample.getOredCriteria().get(0).andNameLike('%' + pluginQueryReq.getName() + '%'); + queryWrapper.lambda().like(PluginDO::getName, pluginQueryReq.getName()); } if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) { - pluginDOExample.getOredCriteria().get(0).andPatternLike('%' + pluginQueryReq.getPattern() + '%'); + queryWrapper.lambda().like(PluginDO::getPattern, pluginQueryReq.getPattern()); } if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) { - pluginDOExample.getOredCriteria().get(0).andCreatedByEqualTo(pluginQueryReq.getCreatedBy()); + queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy()); } - List pluginDOS = pluginRepository.query(pluginDOExample); + List pluginDOS = pluginRepository.query(queryWrapper); if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) { pluginDOS = pluginDOS.stream().filter(pluginDO -> pluginDO.getPattern().contains(pluginQueryReq.getPattern()) @@ -175,29 +172,16 @@ public class PluginServiceImpl implements PluginService { }, a -> a, (k1, k2) -> k1)); } + //todo private List authCheck(List plugins, User user) { - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - List modelIdAuthorized = semanticInterpreter.getModelList(AuthType.ADMIN, null, user).stream() - .map(ModelResp::getId).collect(Collectors.toList()); - plugins = plugins.stream().filter(plugin -> { - if (CollectionUtils.isEmpty(plugin.getModelList()) || plugin.isContainsAllModel()) { - return true; - } - for (Long modelId : plugin.getModelList()) { - if (modelIdAuthorized.contains(modelId)) { - return true; - } - } - return false; - }).collect(Collectors.toList()); return plugins; } public Plugin convert(PluginDO pluginDO) { Plugin plugin = new Plugin(); BeanUtils.copyProperties(pluginDO, plugin); - if (StringUtils.isNotBlank(pluginDO.getModel())) { - plugin.setModelList(Arrays.stream(pluginDO.getModel().split(",")) + if (StringUtils.isNotBlank(pluginDO.getView())) { + plugin.setViewList(Arrays.stream(pluginDO.getView().split(",")) .map(Long::parseLong).collect(Collectors.toList())); } return plugin; @@ -210,7 +194,7 @@ public class PluginServiceImpl implements PluginService { pluginDO.setCreatedBy(user.getName()); pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); - pluginDO.setModel(StringUtils.join(plugin.getModelList(), ",")); + pluginDO.setView(StringUtils.join(plugin.getViewList(), ",")); return pluginDO; } @@ -218,7 +202,7 @@ public class PluginServiceImpl implements PluginService { BeanUtils.copyProperties(plugin, pluginDO); pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); - pluginDO.setModel(StringUtils.join(plugin.getModelList(), ",")); + pluginDO.setView(StringUtils.join(plugin.getViewList(), ",")); return pluginDO; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index c0abbac30..014638f89 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; @@ -65,14 +66,6 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; @@ -97,6 +90,15 @@ import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Service @Component("chatQueryService") @Primary @@ -290,7 +292,7 @@ public class QueryServiceImpl implements QueryService { similarQueryManager.saveSimilarQuery(SimilarQueryReq.builder().parseId(queryReq.getParseId()) .queryId(queryReq.getQueryId()) .agentId(chatQueryDO.getAgentId()) - .modelId(parseInfo.getModelClusterKey()) + .viewId(parseInfo.getViewId()) .queryText(queryReq.getQueryText()).build()); } @@ -346,8 +348,9 @@ public class QueryServiceImpl implements QueryService { } QueryResult queryResult = semanticQuery.execute(user); queryResult.setChatContext(semanticQuery.getParseInfo()); + ViewSchema viewSchema = semanticSchema.getViewSchemaMap().get(parseInfo.getViewId()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, user); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, viewSchema, user); queryResult.setEntityInfo(entityInfo); return queryResult; } @@ -415,7 +418,8 @@ public class QueryServiceImpl implements QueryService { ChatParseDO chatParseDO = chatService.getParseInfo(queryId, parseId); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - return semanticService.getEntityInfo(parseInfo, user); + ViewSchema viewSchema = schemaService.getSemanticSchema().getViewSchemaMap().get(parseInfo.getViewId()); + return semanticService.getEntityInfo(parseInfo, viewSchema, user); } private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo, @@ -632,10 +636,10 @@ public class QueryServiceImpl implements QueryService { SemanticQueryResp semanticQueryResp = new SemanticQueryResp(); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticSchema semanticSchema = semanticService.getSemanticSchema(); - SchemaElement schemaElement = semanticSchema.getDimensions(dimensionValueReq.getElementID()); + SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID()); Set detectModelIds = new HashSet<>(); - detectModelIds.add(schemaElement.getModel()); - dimensionValueReq.setModelId(schemaElement.getModel()); + detectModelIds.add(schemaElement.getView()); + dimensionValueReq.setModelId(schemaElement.getView()); List dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds); // if the search results is null,search dimensionValue from database if (CollectionUtils.isEmpty(dimensionValues)) { @@ -692,7 +696,7 @@ public class QueryServiceImpl implements QueryService { dateConf.setPeriod("DAY"); queryStructReq.setDateInfo(dateConf); queryStructReq.setLimit(20L); - queryStructReq.addModelId(dimensionValueReq.getModelId()); + queryStructReq.setViewId(dimensionValueReq.getModelId()); queryStructReq.setQueryType(QueryType.ID); List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java index cf1e222d9..6c9a1c105 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.RecommendReq; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; @@ -13,6 +13,11 @@ import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.service.RecommendService; import com.tencent.supersonic.chat.server.service.SemanticService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -20,10 +25,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; /*** * Recommend Service impl @@ -47,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService { if (Objects.isNull(modelId)) { return new RecommendResp(); } - ModelSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema modelSchema = semanticService.getModelSchema(modelId); if (Objects.isNull(modelSchema)) { return new RecommendResp(); } @@ -79,7 +80,7 @@ public class RecommendServiceImpl implements RecommendService { .limit(limit) .map(dimSchemaDesc -> { SchemaElement item = new SchemaElement(); - item.setModel(modelId); + item.setView(modelId); item.setName(dimSchemaDesc.getName()); item.setBizName(dimSchemaDesc.getBizName()); item.setId(dimSchemaDesc.getId()); @@ -93,7 +94,7 @@ public class RecommendServiceImpl implements RecommendService { .limit(limit) .map(metricSchemaDesc -> { SchemaElement item = new SchemaElement(); - item.setModel(modelId); + item.setView(modelId); item.setName(metricSchemaDesc.getName()); item.setBizName(metricSchemaDesc.getBizName()); item.setId(metricSchemaDesc.getId()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java index 5e1509dcf..4df5d17dc 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java @@ -3,14 +3,15 @@ package com.tencent.supersonic.chat.server.service.impl; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; + @Service @Slf4j public class SchemaService { @@ -27,13 +28,13 @@ public class SchemaService { @Override public SemanticSchema load(String key) { log.info("load getDomainSchemaInfo cache [{}]", key); - return new SemanticSchema(semanticInterpreter.getModelSchema()); + return new SemanticSchema(semanticInterpreter.getViewSchema()); } } ); - public ModelSchema getModelSchema(Long id) { - return semanticInterpreter.getModelSchema(id, true); + public ViewSchema getViewSchema(Long id) { + return semanticInterpreter.getViewSchema(id, true); } public SemanticSchema getSemanticSchema() { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java index a4b54fa15..fdf56324c 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.server.service.impl; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.Lists; import com.hankcs.hanlp.seg.common.Term; -import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; @@ -15,11 +14,12 @@ import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult; -import com.tencent.supersonic.chat.core.knowledge.ModelInfoStat; +import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat; import com.tencent.supersonic.chat.core.mapper.MapperHelper; import com.tencent.supersonic.chat.core.mapper.MatchText; import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType; import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy; +import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.core.utils.HanlpHelper; import com.tencent.supersonic.chat.core.utils.NatureHelper; import com.tencent.supersonic.chat.server.service.AgentService; @@ -28,6 +28,13 @@ import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.service.SearchService; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -40,12 +47,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; /** @@ -86,13 +87,13 @@ public class SearchServiceImpl implements SearchService { // 2.get meta info SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema(); List metricsDb = semanticSchemaDb.getMetrics(); - final Map modelToName = semanticSchemaDb.getModelIdToName(); + final Map modelToName = semanticSchemaDb.getViewIdToName(); // 3.detect by segment List originals = HanlpHelper.getTerms(queryText); log.info("hanlp parse result: {}", originals); MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); - Set detectModelIds = mapperHelper.getModelIds(queryReq.getModelId(), agentService.getAgent(agentId)); + Set detectModelIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId)); QueryContext queryContext = new QueryContext(); BeanUtils.copyProperties(queryReq, queryContext); @@ -116,7 +117,7 @@ public class SearchServiceImpl implements SearchService { log.info("searchTextEntry:{},queryReq:{}", searchTextEntry, queryReq); Set searchResults = new LinkedHashSet(); - ModelInfoStat modelStat = NatureHelper.getModelStat(originals); + ViewInfoStat modelStat = NatureHelper.getViewStat(originals); List possibleModels = getPossibleModels(queryReq, originals, modelStat, queryReq.getModelId()); @@ -132,7 +133,7 @@ public class SearchServiceImpl implements SearchService { for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) { Set searchResultSet = searchDimensionValue(metricsDb, modelToName, - modelStat.getMetricModelCount(), existMetricAndDimension, + modelStat.getMetricViewCount(), existMetricAndDimension, matchText, natureToNameMap, natureToNameEntry, queryReq.getQueryFilters()); searchResults.addAll(searchResultSet); @@ -141,7 +142,7 @@ public class SearchServiceImpl implements SearchService { } private List getPossibleModels(QueryReq queryCtx, List originals, - ModelInfoStat modelStat, Long webModelId) { + ViewInfoStat modelStat, Long webModelId) { if (Objects.nonNull(webModelId) && webModelId > 0) { List result = new ArrayList<>(); @@ -149,22 +150,22 @@ public class SearchServiceImpl implements SearchService { return result; } - List possibleModels = NatureHelper.selectPossibleModels(originals); + List possibleModels = NatureHelper.selectPossibleViews(originals); - Set contextModel = chatService.getContextModel(queryCtx.getChatId()); + Long contextModel = chatService.getContextModel(queryCtx.getChatId()); log.debug("possibleModels:{},modelStat:{},contextModel:{}", possibleModels, modelStat, contextModel); // If nothing is recognized or only metric are present, then add the contextModel. if (nothingOrOnlyMetric(modelStat)) { - return contextModel.stream().filter(modelId -> modelId > 0).collect(Collectors.toList()); + return Lists.newArrayList(contextModel); } return possibleModels; } - private boolean nothingOrOnlyMetric(ModelInfoStat modelStat) { - return modelStat.getMetricModelCount() >= 0 && modelStat.getDimensionModelCount() <= 0 - && modelStat.getDimensionValueModelCount() <= 0 && modelStat.getModelCount() <= 0; + private boolean nothingOrOnlyMetric(ViewInfoStat modelStat) { + return modelStat.getMetricViewCount() >= 0 && modelStat.getDimensionViewCount() <= 0 + && modelStat.getDimensionValueViewCount() <= 0 && modelStat.getViewCount() <= 0; } private boolean effectiveModel(Long contextModel) { @@ -184,7 +185,7 @@ public class SearchServiceImpl implements SearchService { String nature = natureToNameEntry.getKey(); String wordName = natureToNameEntry.getValue(); - Long modelId = NatureHelper.getModelId(nature); + Long modelId = NatureHelper.getViewId(nature); SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature); if (SchemaElementType.ENTITY.equals(schemaElementType)) { @@ -261,7 +262,7 @@ public class SearchServiceImpl implements SearchService { return Lists.newArrayList(); } return metricsDb.stream() - .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getModel())) + .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getView())) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .flatMap(entry -> { List result = new ArrayList<>(); @@ -285,7 +286,7 @@ public class SearchServiceImpl implements SearchService { if (CollectionUtils.isEmpty(possibleModels)) { return true; } - Long model = NatureHelper.getModelId(nature); + Long model = NatureHelper.getViewId(nature); return possibleModels.contains(model); }) .map(nature -> { @@ -308,7 +309,7 @@ public class SearchServiceImpl implements SearchService { for (HanlpMapResult hanlpMapResult : hanlpMapResults) { List dimensionMetricClassIds = hanlpMapResult.getNatures().stream() - .map(nature -> new ModelWithSemanticType(NatureHelper.getModelId(nature), + .map(nature -> new ModelWithSemanticType(NatureHelper.getViewId(nature), NatureHelper.convertToElementType(nature))) .filter(entry -> matchCondition(entry, possibleModels)).collect(Collectors.toList()); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java index 650447e4e..b8c823175 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java @@ -1,17 +1,18 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.knowledge.DictWord; import com.tencent.supersonic.chat.core.knowledge.builder.WordBuilderFactory; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DictWordType; -import java.util.ArrayList; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; + @Service @Slf4j @@ -21,13 +22,13 @@ public class WordService { public List getAllDictWords() { SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getModelSchema()); + SemanticSchema semanticSchema = new SemanticSchema(semanticInterpreter.getViewSchema()); List words = new ArrayList<>(); addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words); addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words); - addWordsByType(DictWordType.MODEL, semanticSchema.getModels(), words); + addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words); addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words); addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java index 2a8ef82fa..3da5ab77a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.util; import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; @@ -56,7 +56,7 @@ public class ChatConfigHelper { return chatConfig; } - public List generateAllDimIdList(ModelSchema modelSchema) { + public List generateAllDimIdList(ViewSchema modelSchema) { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getDimensions())) { return new ArrayList<>(); } @@ -65,7 +65,7 @@ public class ChatConfigHelper { return new ArrayList<>(dimIdAndDescPair.keySet()); } - public List generateAllMetricIdList(ModelSchema modelSchema) { + public List generateAllMetricIdList(ViewSchema modelSchema) { if (Objects.isNull(modelSchema) || CollectionUtils.isEmpty(modelSchema.getMetrics())) { return new ArrayList<>(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java index 18bf00e21..c19930e5b 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java @@ -1,11 +1,8 @@ package com.tencent.supersonic.chat.server.util; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; - import com.github.pagehelper.PageInfo; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; @@ -19,6 +16,13 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO; import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -28,12 +32,9 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; @Component public class DictMetaHelper { @@ -56,13 +57,13 @@ public class DictMetaHelper { dimValueDOList = generateDimValueInfoByModel(modelIds); break; case OFFLINE_FULL: - List modelSchemaDescList = semanticInterpreter.getModelSchema(); + List modelSchemaDescList = semanticInterpreter.getViewSchema(); if (CollectionUtils.isEmpty(modelSchemaDescList)) { break; } - Map modelIdAndDescPair = modelSchemaDescList.stream() - .collect(Collectors.toMap(a -> a.getModel().getId(), schema -> schema, (k1, k2) -> k1)); + Map modelIdAndDescPair = modelSchemaDescList.stream() + .collect(Collectors.toMap(a -> a.getView().getId(), schema -> schema, (k1, k2) -> k1)); if (!CollectionUtils.isEmpty(modelIdAndDescPair)) { modelIds.addAll(modelIdAndDescPair.keySet()); dimValueDOList = generateDimValueInfoByModel(modelIds); @@ -87,12 +88,12 @@ public class DictMetaHelper { return dimValueDOList; } - List modelSchemaDescList = semanticInterpreter.getModelSchema(); + List modelSchemaDescList = semanticInterpreter.getViewSchema(); if (CollectionUtils.isEmpty(modelSchemaDescList)) { return dimValueDOList; } - Map modelIdAndDescPair = modelSchemaDescList.stream() - .collect(Collectors.toMap(a -> a.getModel().getId(), a -> a, (k1, k2) -> k1)); + Map modelIdAndDescPair = modelSchemaDescList.stream() + .collect(Collectors.toMap(a -> a.getView().getId(), a -> a, (k1, k2) -> k1)); for (Long modelId : modelAndDimMap.keySet()) { if (!modelIdAndDescPair.containsKey(modelId)) { @@ -116,7 +117,7 @@ public class DictMetaHelper { private List generateDimValueInfoByModel(Set modelIds) { List dimValueDOList = new ArrayList<>(); - List modelSchemaDescList = semanticInterpreter.getModelSchema(new ArrayList<>(modelIds)); + List modelSchemaDescList = semanticInterpreter.getViewSchema(new ArrayList<>(modelIds)); if (CollectionUtils.isEmpty(modelSchemaDescList)) { return dimValueDOList; } @@ -124,7 +125,7 @@ public class DictMetaHelper { modelSchemaDescList.forEach(modelSchemaDesc -> { Map dimIdAndDescPair = modelSchemaDesc.getDimensions().stream() .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1)); - fillDimValueDOList(dimValueDOList, modelSchemaDesc.getModel().getId(), dimIdAndDescPair); + fillDimValueDOList(dimValueDOList, modelSchemaDesc.getView().getId(), dimIdAndDescPair); }); diff --git a/chat/server/src/main/resources/mapper/PluginDOMapper.xml b/chat/server/src/main/resources/mapper/PluginDOMapper.xml deleted file mode 100644 index 37a9ef60b..000000000 --- a/chat/server/src/main/resources/mapper/PluginDOMapper.xml +++ /dev/null @@ -1,276 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, type, model, pattern, parse_mode, name, created_at, created_by, updated_at, - updated_by - - - parse_mode_config, config, comment - - - - - - delete from s2_plugin - where id = #{id,jdbcType=BIGINT} - - - insert into s2_plugin (id, type, model, - pattern, parse_mode, name, - created_at, created_by, updated_at, - updated_by, parse_mode_config, config, - comment) - values (#{id,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{model,jdbcType=VARCHAR}, - #{pattern,jdbcType=VARCHAR}, #{parseMode,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, - #{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, - #{updatedBy,jdbcType=VARCHAR}, #{parseModeConfig,jdbcType=LONGVARCHAR}, #{config,jdbcType=LONGVARCHAR}, - #{comment,jdbcType=LONGVARCHAR}) - - - insert into s2_plugin - - - id, - - - type, - - - model, - - - pattern, - - - parse_mode, - - - name, - - - created_at, - - - created_by, - - - updated_at, - - - updated_by, - - - parse_mode_config, - - - config, - - - comment, - - - - - #{id,jdbcType=BIGINT}, - - - #{type,jdbcType=VARCHAR}, - - - #{model,jdbcType=VARCHAR}, - - - #{pattern,jdbcType=VARCHAR}, - - - #{parseMode,jdbcType=VARCHAR}, - - - #{name,jdbcType=VARCHAR}, - - - #{createdAt,jdbcType=TIMESTAMP}, - - - #{createdBy,jdbcType=VARCHAR}, - - - #{updatedAt,jdbcType=TIMESTAMP}, - - - #{updatedBy,jdbcType=VARCHAR}, - - - #{parseModeConfig,jdbcType=LONGVARCHAR}, - - - #{config,jdbcType=LONGVARCHAR}, - - - #{comment,jdbcType=LONGVARCHAR}, - - - - - - update s2_plugin - - - type = #{type,jdbcType=VARCHAR}, - - - model = #{model,jdbcType=VARCHAR}, - - - pattern = #{pattern,jdbcType=VARCHAR}, - - - parse_mode = #{parseMode,jdbcType=VARCHAR}, - - - name = #{name,jdbcType=VARCHAR}, - - - created_at = #{createdAt,jdbcType=TIMESTAMP}, - - - created_by = #{createdBy,jdbcType=VARCHAR}, - - - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - - - updated_by = #{updatedBy,jdbcType=VARCHAR}, - - - parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR}, - - - config = #{config,jdbcType=LONGVARCHAR}, - - - comment = #{comment,jdbcType=LONGVARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update s2_plugin - set type = #{type,jdbcType=VARCHAR}, - model = #{model,jdbcType=VARCHAR}, - pattern = #{pattern,jdbcType=VARCHAR}, - parse_mode = #{parseMode,jdbcType=VARCHAR}, - name = #{name,jdbcType=VARCHAR}, - created_at = #{createdAt,jdbcType=TIMESTAMP}, - created_by = #{createdBy,jdbcType=VARCHAR}, - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - updated_by = #{updatedBy,jdbcType=VARCHAR}, - parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR}, - config = #{config,jdbcType=LONGVARCHAR}, - comment = #{comment,jdbcType=LONGVARCHAR} - where id = #{id,jdbcType=BIGINT} - - - update s2_plugin - set type = #{type,jdbcType=VARCHAR}, - model = #{model,jdbcType=VARCHAR}, - pattern = #{pattern,jdbcType=VARCHAR}, - parse_mode = #{parseMode,jdbcType=VARCHAR}, - name = #{name,jdbcType=VARCHAR}, - created_at = #{createdAt,jdbcType=TIMESTAMP}, - created_by = #{createdBy,jdbcType=VARCHAR}, - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - updated_by = #{updatedBy,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java index 67cb6cd9b..f4be2dccc 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.processor; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; @@ -115,7 +115,7 @@ class MetricCheckProcessorTest { * 访问用户数 drill down dimension is 部门, and 部门 is necessary, 部门 need in select and group by or where expressions */ private SemanticSchema mockModelSchema() { - ModelSchema modelSchema = new ModelSchema(); + ViewSchema modelSchema = new ViewSchema(); Set metrics = Sets.newHashSet( mockElement(1L, "访问次数", SchemaElementType.METRIC, Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(false).build(), @@ -129,7 +129,7 @@ class MetricCheckProcessorTest { } private SemanticSchema mockModelSchemaNoDimensionSetting() { - ModelSchema modelSchema = new ModelSchema(); + ViewSchema modelSchema = new ViewSchema(); Set metrics = Sets.newHashSet( mockElement(1L, "访问次数", SchemaElementType.METRIC, Lists.newArrayList()), mockElement(2L, "访问用户数", SchemaElementType.METRIC, Lists.newArrayList()) diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index 916ed7ec7..6a6e222f6 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -5,21 +5,22 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * QueryReqBuilderTest */ @@ -29,9 +30,9 @@ class QueryReqBuilderTest { void buildS2SQLReq() { init(); QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.addModelId(1L); + queryStructReq.setViewId(1L); + queryStructReq.setViewName("内容库"); queryStructReq.setQueryType(QueryType.METRIC); - queryStructReq.setModelName("内容库"); Aggregator aggregator = new Aggregator(); aggregator.setFunc(AggOperatorEnum.UNKNOWN); @@ -51,18 +52,18 @@ class QueryReqBuilderTest { orders.add(order); queryStructReq.setOrders(orders); - QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " - + "BY department ORDER BY uv LIMIT 2000", querySqlReq.getSql()); + + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); queryStructReq.setQueryType(QueryType.TAG); - querySqlReq = queryStructReq.convert(queryStructReq); + querySQLReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", - querySqlReq.getSql()); + querySQLReq.getSql()); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java deleted file mode 100644 index 8070b65f4..000000000 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.tencent.supersonic.common.pojo; - -import lombok.Data; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Data -public class ModelCluster { - - private static final String split = "_"; - - private Set modelIds = new LinkedHashSet<>(); - - private Set modelNames = new LinkedHashSet<>(); - - private String key; - - private String name; - - public static ModelCluster build(Set modelIds) { - ModelCluster modelCluster = new ModelCluster(); - modelCluster.setModelIds(modelIds); - modelCluster.setKey(StringUtils.join(modelIds, split)); - return modelCluster; - } - - public static ModelCluster build(String key) { - ModelCluster modelCluster = new ModelCluster(); - modelCluster.setModelIds(getModelIdFromKey(key)); - modelCluster.setKey(key); - return modelCluster; - } - - public void buildName(Map modelNameMap) { - modelNames = modelNameMap.entrySet().stream().filter(entry -> - modelIds.contains(entry.getKey())).map(Map.Entry::getValue) - .collect(Collectors.toSet()); - name = String.join(split, modelNames); - } - - public static Set getModelIdFromKey(String key) { - return Arrays.stream(key.split(split)) - .map(Long::parseLong).collect(Collectors.toSet()); - } - - public Long getFirstModel() { - if (CollectionUtils.isEmpty(modelIds)) { - return -1L; - } - return new ArrayList<>(modelIds).get(0); - } - -} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java index 1948fa175..12c7b9cd9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java @@ -14,7 +14,7 @@ public enum DictWordType { VALUE("value"), - MODEL("model"), + VIEW("view"), ENTITY("entity"), @@ -48,7 +48,7 @@ public enum DictWordType { //domain String[] natures = nature.split(DictWordType.NATURE_SPILT); if (natures.length == 2 && StringUtils.isNumeric(natures[1])) { - return MODEL; + return VIEW; } //dimension value if (natures.length == 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java new file mode 100644 index 000000000..578768655 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.common.pojo.enums; + +public enum TimeMode { + + /** + * date mode + * LAST - a certain time + * RECENT - a period time + */ + LAST, RECENT +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java index 7f83e4fc1..40b3d84ca 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java @@ -1,32 +1,13 @@ package com.tencent.supersonic.common.pojo.enums; - public enum TypeEnums { - DATASOURCE("datasource"), - METRIC("metric"), - DIMENSION("dimension"), - DOMAIN("domain"), - ENTITY("entity"), - UNKNOWN("unknown"); + METRIC, + DIMENSION, + DOMAIN, + ENTITY, + VIEW, + MODEL, + UNKNOWN; - - private String name; - - TypeEnums(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static TypeEnums of(String type) { - for (TypeEnums typeEnum : TypeEnums.values()) { - if (typeEnum.name.equalsIgnoreCase(type)) { - return typeEnum; - } - } - return TypeEnums.UNKNOWN; - } } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java index f7c3dc86d..487c4c9b0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/SignatureUtils.java @@ -1,33 +1,15 @@ package com.tencent.supersonic.common.util; import com.tencent.supersonic.common.pojo.Pair; -import org.apache.commons.codec.binary.Hex; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; - -import static java.lang.Thread.sleep; +import org.apache.commons.codec.digest.DigestUtils; public class SignatureUtils { - private static final String ALGORITHM_HMAC_SHA256 = "HmacSHA256"; - private static final long TIME_OUT = 60 * 1000 * 30; - public static String generateSignature(String appKey, String appSecret, long timestamp) { - try { - Mac sha256HMAC = Mac.getInstance(ALGORITHM_HMAC_SHA256); - SecretKeySpec secretKey = new SecretKeySpec(appSecret.getBytes(), ALGORITHM_HMAC_SHA256); - sha256HMAC.init(secretKey); - - String data = appKey + timestamp; - byte[] hash = sha256HMAC.doFinal(data.getBytes()); - - return Hex.encodeHexString(hash); - } catch (NoSuchAlgorithmException | InvalidKeyException e) { - throw new RuntimeException("Error generating signature", e); - } + public static String generateSignature(String appSecret, long timestamp) { + String psw = timestamp + appSecret + timestamp; + return DigestUtils.sha1Hex(psw); } public static Pair isValidSignature(String appKey, String appSecret, @@ -42,7 +24,7 @@ public class SignatureUtils { return new Pair<>(false, "Timestamp is too old"); } - String generatedSignature = generateSignature(appKey, appSecret, timestamp); + String generatedSignature = generateSignature(appSecret, timestamp); if (generatedSignature.equals(signatureToCheck)) { return new Pair<>(true, "Signature is valid"); @@ -52,19 +34,15 @@ public class SignatureUtils { } public static void main(String[] args) throws InterruptedException { - // appkey为申请的接口id - String appKey = "1"; //生成的密钥 - String appSecret = "8fb44f17-f37d-4510-bb29-59b0e0b266d0"; - long timestamp = System.currentTimeMillis(); + String appSecret = "38f2857c-d9ee-4c3a-bcc2-2cdb62fda5aa"; + long timestamp = 1706504908126L; System.out.println("timeStamp:" + timestamp); //生成的签名 - String serverSignature = generateSignature(appKey, appSecret, timestamp); + String serverSignature = generateSignature(appSecret, timestamp); System.out.println("Server Signature: " + serverSignature); - - sleep(4000); //用户需要的入参 - Pair isValid = isValidSignature(appKey, appSecret, timestamp, serverSignature); + Pair isValid = isValidSignature("1", appSecret, timestamp, serverSignature); System.out.println("Is Signature Valid? " + isValid.first); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java index 8b8eb2671..218b24f01 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AppConfig.java @@ -3,22 +3,11 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.collect.Lists; import lombok.Data; -import java.util.HashSet; import java.util.List; -import java.util.Set; @Data public class AppConfig { private List items = Lists.newArrayList(); - public Set getAllItems() { - Set itemSet = new HashSet<>(); - for (Item item : items) { - itemSet.add(item); - itemSet.addAll(item.getRelateItems()); - } - return itemSet; - } - } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java index c545ebd89..1f39f9813 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DrillDownDimension.java @@ -13,7 +13,15 @@ public class DrillDownDimension { private boolean necessary; + private boolean inheritedFromModel; + public DrillDownDimension(Long dimensionId) { this.dimensionId = dimensionId; } + + public DrillDownDimension(Long dimensionId, boolean necessary) { + this.dimensionId = dimensionId; + this.necessary = necessary; + } + } \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java new file mode 100644 index 000000000..72eb229fd --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java @@ -0,0 +1,10 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; + +@Data +public class MetricTypeDefaultConfig { + + private TimeDefaultConfig timeDefaultConfig; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java new file mode 100644 index 000000000..1f077e2ea --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryConfig.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; + +@Data +public class QueryConfig { + + private TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig(); + + private MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); + + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java index 82965f8f2..9329182e8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryStat.java @@ -2,12 +2,12 @@ package com.tencent.supersonic.headless.api.pojo; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; -import java.util.List; -import java.util.Objects; - import lombok.Data; import lombok.ToString; +import java.util.List; +import java.util.Objects; + @Data @ToString public class QueryStat { @@ -15,6 +15,8 @@ public class QueryStat { private Long id; private String traceId; private Long modelId; + + private Long viewId; private String user; private String createdAt; /** @@ -91,6 +93,11 @@ public class QueryStat { return this; } + public QueryStat setViewId(Long viewId) { + this.viewId = viewId; + return this; + } + public QueryStat setUser(String user) { this.user = user; return this; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java new file mode 100644 index 000000000..f6a61ee61 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class TagTypeDefaultConfig { + + //When displaying tag selection results, the information displayed by default + private List dimensionIds = new ArrayList<>(); + private List metricIds = new ArrayList<>(); + + //default time to filter tag selection results + private TimeDefaultConfig timeDefaultConfig; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java new file mode 100644 index 000000000..d0eb71b07 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TimeDefaultConfig.java @@ -0,0 +1,26 @@ +package com.tencent.supersonic.headless.api.pojo; + + +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import lombok.Data; + +@Data +public class TimeDefaultConfig { + + + /** + * default time span unit + */ + private Integer unit = 1; + + /** + * default time type: day + * DAY, WEEK, MONTH, YEAR + */ + private String period = Constants.DAY; + + private TimeMode timeMode = TimeMode.LAST; + + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java new file mode 100644 index 000000000..9e39b73c9 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewDetail.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; +import java.util.List; + +@Data +public class ViewDetail { + + private List viewModelConfigs; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java new file mode 100644 index 000000000..4f374dd95 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ViewModelConfig.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.headless.api.pojo; + +import com.google.common.collect.Lists; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ViewModelConfig { + + private Long id; + + private boolean includesAll; + + private List metrics = Lists.newArrayList(); + + private List dimensions = Lists.newArrayList(); + + public ViewModelConfig(Long id, boolean includesAll) { + this.id = id; + this.includesAll = includesAll; + } +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java new file mode 100644 index 000000000..e3bbfda41 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/SchemaType.java @@ -0,0 +1,7 @@ +package com.tencent.supersonic.headless.api.pojo.enums; + +public enum SchemaType { + + VIEW, + MODEL +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewInfoReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java similarity index 92% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewInfoReq.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java index e1777b0c3..c422c6fc7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewInfoReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/CanvasReq.java @@ -1,10 +1,11 @@ package com.tencent.supersonic.headless.api.pojo.request; -import java.util.Date; import lombok.Data; +import java.util.Date; + @Data -public class ViewInfoReq { +public class CanvasReq { private Long id; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java index 36f10d830..ce8f41d54 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricQueryReq.java @@ -11,7 +11,6 @@ public class MetricQueryReq { private List metrics; private List dimensions; - private String rootPath = ""; private Map variables; private String where; private Long limit; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelSchemaFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelSchemaFilterReq.java deleted file mode 100644 index b9f27e052..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelSchemaFilterReq.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo.request; - -import lombok.Data; - -import java.util.List; - -@Data -public class ModelSchemaFilterReq { - - /** - * if domainIds is empty, get all domain info - */ - private List modelIds; -} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java index f2d121c33..ec4519030 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java @@ -1,15 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.tencent.supersonic.headless.api.pojo.MetricTable; +import lombok.Data; + import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.Data; @Data public class ParseSqlReq { - - private String rootPath = ""; private Map variables; private String sql = ""; private List tables; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java index 3ad0b4f40..f37caf3dd 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMultiStructReq.java @@ -2,13 +2,13 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.headless.api.pojo.Cache; -import java.util.ArrayList; -import java.util.List; import lombok.Data; import lombok.ToString; import org.apache.commons.codec.digest.DigestUtils; import org.springframework.util.CollectionUtils; +import java.util.List; + @Data @ToString public class QueryMultiStructReq extends SemanticQueryReq { @@ -23,11 +23,11 @@ public class QueryMultiStructReq extends SemanticQueryReq { return DigestUtils.md5Hex(this.toCustomizedString()); } - public List getModelIds() { + public Long getViewId() { if (CollectionUtils.isEmpty(this.getQueryStructReqs())) { - return new ArrayList<>(); + return null; } - return this.getQueryStructReqs().get(0).getModelIds(); + return this.getQueryStructReqs().get(0).getViewId(); } public Cache getCacheInfo() { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java index ddd1128a9..ddac40577 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java @@ -12,7 +12,9 @@ public class QuerySqlReq extends SemanticQueryReq { @Override public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"modelId\":") + stringBuilder.append("\"viewId\":") + .append(viewId); + stringBuilder.append("\"modelIds\":") .append(modelIds); stringBuilder.append(",\"params\":") .append(params); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 274caaa55..a2cdb275b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -44,8 +44,6 @@ import java.util.stream.Collectors; @Data @Slf4j public class QueryStructReq extends SemanticQueryReq { - - private String modelName; private List groups = new ArrayList<>(); private List aggregators = new ArrayList<>(); private List orders = new ArrayList<>(); @@ -94,7 +92,9 @@ public class QueryStructReq extends SemanticQueryReq { public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); - stringBuilder.append("\"modelId\":") + stringBuilder.append("\"viewId\":") + .append(viewId); + stringBuilder.append("\"modelIds\":") .append(modelIds); stringBuilder.append(",\"groups\":") .append(groups); @@ -125,7 +125,9 @@ public class QueryStructReq extends SemanticQueryReq { @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); - sb.append("\"modelId\":") + sb.append("\"viewId\":") + .append(viewId); + sb.append("\"modelIds\":") .append(modelIds); sb.append(",\"groups\":") .append(groups); @@ -169,6 +171,7 @@ public class QueryStructReq extends SemanticQueryReq { QuerySqlReq result = new QuerySqlReq(); result.setSql(sql); + result.setViewId(queryStructReq.getViewId()); result.setModelIds(queryStructReq.getModelIdSet()); result.setParams(new ArrayList<>()); return result; @@ -211,7 +214,7 @@ public class QueryStructReq extends SemanticQueryReq { } plainSelect.setSelectItems(selectItems); //2.Set the table name - Table table = new Table(queryStructReq.getModelName()); + Table table = new Table(queryStructReq.getTableName()); plainSelect.setFromItem(table); //3.Set the order by clause @@ -271,9 +274,14 @@ public class QueryStructReq extends SemanticQueryReq { return sql; } - public String getModelName() { - return Objects.nonNull(modelName) ? modelName : - Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_"); + public String getTableName() { + if (StringUtils.isNotBlank(viewName)) { + return viewName; + } + if (viewId != null) { + return Constants.TABLE_PREFIX + viewId; + } + return Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_"); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java new file mode 100644 index 000000000..bb08c24c8 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SchemaFilterReq.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.google.common.collect.Lists; +import lombok.Data; + +import java.util.List; + +@Data +public class SchemaFilterReq { + + private Long viewId; + + private List modelIds = Lists.newArrayList(); + +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index d424d4928..a9a2dedd5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -3,14 +3,14 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Param; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; @Data @@ -19,7 +19,12 @@ public abstract class SemanticQueryReq { protected boolean needAuth = true; + protected Long viewId; + + protected String viewName; + protected Set modelIds = new HashSet<>(); + protected List params = new ArrayList<>(); protected Cache cacheInfo = new Cache(); @@ -38,10 +43,6 @@ public abstract class SemanticQueryReq { return Lists.newArrayList(modelIds); } - public String getModelIdStr() { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); - } - public Set getModelIdSet() { return modelIds; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java new file mode 100644 index 000000000..af113b435 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewFilterReq.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.google.common.collect.Lists; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +@Data +@NoArgsConstructor +public class ViewFilterReq { + + protected boolean defaultAll = true; + + protected List viewIds = Lists.newArrayList(); + + public ViewFilterReq(Long viewId) { + addView(viewId); + } + + public List getViewIds() { + if (CollectionUtils.isEmpty(viewIds) && !defaultAll) { + return Lists.newArrayList(-1L); + } + return viewIds; + } + + public void addView(Long viewId) { + viewIds.add(viewId); + } + +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java new file mode 100644 index 000000000..09e4fb81a --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ViewReq.java @@ -0,0 +1,41 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import lombok.Data; + +import java.util.List; + +@Data +public class ViewReq extends SchemaItem { + + private Long domainId; + + private ViewDetail viewDetail; + + private String alias; + + private String filterSql; + + private QueryConfig queryConfig; + + private List admins; + + private List adminOrgs; + + public String getAdmin() { + if (admins == null) { + return null; + } + return String.join(",", admins); + } + + public String getAdminOrg() { + if (adminOrgs == null) { + return null; + } + return String.join(",", adminOrgs); + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java index f6fa4f3ce..ce7998942 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/AppResp.java @@ -5,12 +5,15 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.headless.api.pojo.enums.AppStatus; import com.tencent.supersonic.headless.api.pojo.AppConfig; +import com.tencent.supersonic.headless.api.pojo.Item; import lombok.Data; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; @Data public class AppResp extends RecordInfo { @@ -41,4 +44,13 @@ public class AppResp extends RecordInfo { owners = Arrays.asList(owner.split(",")); } + public Set allItems() { + Set itemSet = new HashSet<>(); + for (Item item : config.getItems()) { + itemSet.add(item); + itemSet.addAll(item.getRelateItems()); + } + return itemSet; + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaRelaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java similarity index 88% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaRelaResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java index c52703ca5..9971fb72f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaRelaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/CanvasSchemaResp.java @@ -5,7 +5,7 @@ import lombok.Data; import java.util.List; @Data -public class ModelSchemaRelaResp { +public class CanvasSchemaResp { private Long domainId; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index c6b661c6a..5f81d101f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -29,8 +29,6 @@ public class ModelResp extends SchemaItem { private String depends; - private String sourceType; - private String filterSql; private List viewers = new ArrayList<>(); @@ -43,7 +41,7 @@ public class ModelResp extends SchemaItem { private Integer isOpen; - private List drillDownDimensions; + private List drillDownDimensions = Lists.newArrayList(); private String alias; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java new file mode 100644 index 000000000..3b092c8d4 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -0,0 +1,38 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SemanticSchemaResp { + + private Long viewId; + private List modelIds; + private SchemaType schemaType; + private List metrics = Lists.newArrayList(); + private List dimensions = Lists.newArrayList(); + private List modelRelas = Lists.newArrayList(); + private List modelResps = Lists.newArrayList(); + private ViewResp viewResp; + private DatabaseResp databaseResp; + + public String getSchemaKey() { + if (viewId == null) { + return String.format("%s_%s", schemaType, StringUtils.join(modelIds, UNDERLINE)); + } + return String.format("%s_%s", schemaType, viewId); + + } + +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java new file mode 100644 index 000000000..c6cbd8824 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewResp.java @@ -0,0 +1,61 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; +import lombok.Data; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@Data +public class ViewResp extends SchemaItem { + + private Long domainId; + + private ViewDetail viewDetail; + + private String alias; + + private String filterSql; + + private List admins = new ArrayList<>(); + + private List adminOrgs = new ArrayList<>(); + + private QueryConfig queryConfig; + + public List getAllMetrics() { + return getViewModelConfigs().stream().map(ViewModelConfig::getMetrics) + .flatMap(Collection::stream).collect(Collectors.toList()); + } + + public List getAllDimensions() { + return getViewModelConfigs().stream().map(ViewModelConfig::getMetrics) + .flatMap(Collection::stream).collect(Collectors.toList()); + } + + public List getAllModels() { + return getViewModelConfigs().stream().map(ViewModelConfig::getId) + .collect(Collectors.toList()); + } + + public List getAllIncludeAllModels() { + return getViewModelConfigs().stream().filter(ViewModelConfig::isIncludesAll) + .map(ViewModelConfig::getId) + .collect(Collectors.toList()); + } + + private List getViewModelConfigs() { + if (viewDetail == null || CollectionUtils.isEmpty(viewDetail.getViewModelConfigs())) { + return Lists.newArrayList(); + } + return viewDetail.getViewModelConfigs(); + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java new file mode 100644 index 000000000..5902454a8 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ViewSchemaResp.java @@ -0,0 +1,35 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.Identify; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ViewSchemaResp extends ViewResp { + + private List metrics = Lists.newArrayList(); + private List dimensions = Lists.newArrayList(); + private List modelResps = Lists.newArrayList(); + + public DimSchemaResp getPrimaryKey() { + for (ModelResp modelResp : modelResps) { + Identify identify = modelResp.getPrimaryIdentify(); + if (identify == null) { + continue; + } + for (DimSchemaResp dimension : dimensions) { + if (identify.getBizName().equals(dimension.getBizName())) { + dimension.setEntityAlias(identify.getEntityNames()); + return dimension; + } + } + } + return null; + } +} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java index 145ba1a13..cd8211cbc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java @@ -11,16 +11,17 @@ import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + @Component @Slf4j @Primary @@ -109,10 +110,6 @@ public class DefaultQueryParser implements QueryParser { public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) { MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg); - if (metricQueryReq.getRootPath().isEmpty()) { - queryStatement.setErrMsg("rootPath empty"); - return queryStatement; - } try { return ComponentFactory.getSqlParser().explain(queryStatement, isAgg); } catch (Exception e) { @@ -129,13 +126,12 @@ public class DefaultQueryParser implements QueryParser { metricReq.setDimensions(metricTable.getDimensions()); metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); - metricReq.setRootPath(parseSqlReq.getRootPath()); QueryStatement tableSql = new QueryStatement(); tableSql.setIsS2SQL(false); tableSql.setMetricReq(metricReq); tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); - tableSql.setModelIds(queryStatement.getModelIds()); + tableSql.setViewId(queryStatement.getViewId()); tableSql.setSemanticModel(queryStatement.getSemanticModel()); if (isSingleMetricTable) { tableSql.setViewSql(parseSqlReq.getSql()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java index 80bafdc75..0a2c79cb0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java @@ -6,13 +6,14 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.Objects; + /** * the calcite parse implements */ @@ -50,7 +51,7 @@ public class CalciteSqlParser implements SqlParser { } private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) { - SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build(); + SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); semanticSchema.setSemanticModel(semanticModel); semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDimension(semanticModel.getDimensionMap()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java index e3f5c1a1f..61cb164c6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java @@ -1,18 +1,19 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; import com.tencent.supersonic.headless.core.pojo.Database; +import lombok.Data; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.Data; @Data public class SemanticModel { - private String rootPath; + private String schemaKey; private List metrics = new ArrayList<>(); private Map datasourceMap = new HashMap<>(); private Map> dimensionMap = new HashMap<>(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index f65a3e2dd..34380dd40 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -4,11 +4,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.schema; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.sql.S2SQLSqlValidatorImpl; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.prepare.CalciteCatalogReader; import org.apache.calcite.prepare.Prepare; @@ -19,6 +14,12 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.ParameterScope; import org.apache.calcite.sql.validate.SqlValidatorScope; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + public class SchemaBuilder { public static final String MATERIALIZATION_SYS_DB = "SYS"; @@ -30,10 +31,10 @@ public class SchemaBuilder { public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception { Map nameToTypeMap = new HashMap<>(); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); - rootSchema.add(schema.getRootPath(), schema); + rootSchema.add(schema.getSchemaKey(), schema); Prepare.CatalogReader catalogReader = new CalciteCatalogReader( rootSchema, - Collections.singletonList(schema.getRootPath()), + Collections.singletonList(schema.getSchemaKey()), Configuration.typeFactory, Configuration.config ); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java index cd0486520..2b60cac6e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java @@ -19,7 +19,7 @@ import java.util.Map; public class SemanticSchema extends AbstractSchema { - private final String rootPath; + private final String schemaKey; private final Map tableMap; private SemanticModel semanticModel = new SemanticModel(); @@ -29,17 +29,17 @@ public class SemanticSchema extends AbstractSchema { private RuntimeOptions runtimeOptions; - private SemanticSchema(String rootPath, Map tableMap) { - this.rootPath = rootPath; + private SemanticSchema(String schemaKey, Map tableMap) { + this.schemaKey = schemaKey; this.tableMap = tableMap; } - public static Builder newBuilder(String rootPath) { - return new Builder(rootPath); + public static Builder newBuilder(String schemaKey) { + return new Builder(schemaKey); } - public String getRootPath() { - return rootPath; + public String getSchemaKey() { + return schemaKey; } public void setSemanticModel(SemanticModel semanticModel) { @@ -110,15 +110,15 @@ public class SemanticSchema extends AbstractSchema { public static final class Builder { - private final String rootPath; + private final String schemaKey; private final Map tableMap = new HashMap<>(); - private Builder(String rootPath) { - if (rootPath == null || rootPath.isEmpty()) { + private Builder(String schemaKey) { + if (schemaKey == null) { throw new IllegalArgumentException("Schema name cannot be null or empty"); } - this.rootPath = rootPath; + this.schemaKey = schemaKey; } public Builder addTable(DataSourceTable table) { @@ -132,7 +132,7 @@ public class SemanticSchema extends AbstractSchema { } public SemanticSchema build() { - return new SemanticSchema(rootPath, tableMap); + return new SemanticSchema(schemaKey, tableMap); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index 212925c22..b16c33a31 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -13,14 +13,15 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; /** * supplement the QueryStatement when query with custom aggregation method @@ -49,7 +50,6 @@ public class CalculateAggConverter implements HeadlessConverter { return generateRatioSqlCommand(queryStatement, engineTypeEnum, version); } ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(queryStructReq.getModelIdStr()); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); @@ -111,7 +111,6 @@ public class CalculateAggConverter implements HeadlessConverter { EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setTables(parseSqlReq.getTables()); - sqlCommend.setRootPath(parseSqlReq.getRootPath()); sqlCommend.setVariables(parseSqlReq.getVariables()); sqlCommend.setSupportWith(parseSqlReq.isSupportWith()); } @@ -137,7 +136,6 @@ public class CalculateAggConverter implements HeadlessConverter { check(queryStructReq); queryStatement.setEnableOptimize(false); ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(queryStructReq.getModelIdStr()); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java index 58bdc970d..cc3c028f1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java @@ -63,14 +63,12 @@ public class ParserDefaultConverter implements HeadlessConverter { metricQueryReq.setVariables(queryStructReq.getParams().stream() .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); metricQueryReq.setLimit(queryStructReq.getLimit()); - String rootPath = queryStructReq.getModelIdStr(); - metricQueryReq.setRootPath(rootPath); // support detail query if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) { - Map dataSourceMap = queryStatement.getSemanticModel().getModelMap(); - for (Long modelId : queryStructReq.getModelIds()) { - String modelBizName = dataSourceMap.get(modelId).getName(); + Map modelMap = queryStatement.getSemanticModel().getModelMap(); + for (Long modelId : modelMap.keySet()) { + String modelBizName = modelMap.get(modelId).getName(); String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); metricQueryReq.getMetrics().add(internalMetricName); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index afd34fca9..465e41fd2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -3,15 +3,17 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; -import java.util.List; import lombok.Data; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Triple; +import java.util.List; + @Data public class QueryStatement { - + private Long viewId; private List modelIds; private String sql = ""; private String sourceId = ""; @@ -30,9 +32,10 @@ public class QueryStatement { private String viewSimplifySql = ""; private Boolean enableLimitWrapper = false; - private SemanticModel semanticModel; + private SemanticSchemaResp semanticSchemaResp; + public boolean isOk() { this.ok = "".equals(errMsg) && !"".equals(sql); return ok; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 600346dd4..581277d76 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -1,12 +1,5 @@ package com.tencent.supersonic.headless.core.utils; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; @@ -17,13 +10,23 @@ import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Collections; @@ -35,14 +38,13 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Triple; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; /** * tools functions to analyze queryStructReq @@ -269,19 +271,21 @@ public class SqlGenerateUtils { return modelBizName + UNDERLINE + internalMetricNameSuffix; } - public String generateDerivedMetric(final List metricResps, final Set allFields, - final Map allMeasures, final List dimensionResps, - final String expression, final MetricDefineType metricDefineType, AggOption aggOption, - Set visitedMetric, - Set measures, - Set dimensions) { + public String generateDerivedMetric(final List metricResps, final Set allFields, + final Map allMeasures, + final List dimensionResps, + final String expression, final MetricDefineType metricDefineType, + AggOption aggOption, + Set visitedMetric, + Set measures, + Set dimensions) { Set fields = SqlParserSelectHelper.getColumnFromExpr(expression); if (!CollectionUtils.isEmpty(fields)) { Map replace = new HashMap<>(); for (String field : fields) { switch (metricDefineType) { case METRIC: - Optional metricItem = metricResps.stream() + Optional metricItem = metricResps.stream() .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst(); if (metricItem.isPresent()) { if (visitedMetric.contains(field)) { @@ -302,7 +306,7 @@ public class SqlGenerateUtils { break; case FIELD: if (allFields.contains(field)) { - Optional dimensionItem = dimensionResps.stream() + Optional dimensionItem = dimensionResps.stream() .filter(d -> d.getBizName().equals(field)).findFirst(); if (dimensionItem.isPresent()) { dimensions.add(field); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java index ffaf97c5e..6c160cbaf 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java @@ -16,13 +16,9 @@ import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.server.service.ModelService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -46,11 +42,6 @@ public class AuthCheckBaseAspect { new SimpleDateFormat(Constants.DAY_FORMAT)); @Autowired private AuthService authService; - @Autowired - private DimensionService dimensionService; - @Autowired - private MetricService metricService; - @Autowired private ModelService modelService; @@ -85,18 +76,17 @@ public class AuthCheckBaseAspect { } - public Set getHighSensitiveColsByModelId(List modelIds) { + public Set getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) { Set highSensitiveCols = new HashSet<>(); - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setModelIds(modelIds); - metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); - List highSensitiveDimensions = dimensionService.getDimensions(metaFilter); - List highSensitiveMetrics = metricService.getMetrics(metaFilter); - if (!CollectionUtils.isEmpty(highSensitiveDimensions)) { - highSensitiveDimensions.forEach(dim -> highSensitiveCols.add(dim.getBizName())); + if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) { + semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp -> + SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel())) + .forEach(dim -> highSensitiveCols.add(dim.getBizName())); } - if (!CollectionUtils.isEmpty(highSensitiveMetrics)) { - highSensitiveMetrics.forEach(metric -> highSensitiveCols.add(metric.getBizName())); + if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) { + semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp -> + SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel())) + .forEach(metric -> highSensitiveCols.add(metric.getBizName())); } return highSensitiveCols; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 571e882a9..2d856b1ca 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; @@ -17,13 +18,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -35,6 +29,14 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Aspect @Component @Slf4j @@ -90,7 +92,7 @@ public class DimValueAspect { log.info("correctorSql before replacing:{}", sql); List fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); List dimensions = dimensionService.getDimensions(metaFilter); - Set fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet()); + Set fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet()); Map> filedNameToValueMap = new HashMap<>(); for (FieldExpression expression : fieldExpressionList) { if (!fieldNames.contains(expression.getFieldName())) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index 0f73b6dbe..dc872a6f3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.headless.server.aspect; -import static com.tencent.supersonic.common.pojo.Constants.MINUS; - import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -12,28 +10,20 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.QueryStructUtils; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.StringJoiner; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -48,6 +38,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import static com.tencent.supersonic.common.pojo.Constants.MINUS; @Component @Aspect @@ -108,23 +107,24 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { throws Throwable { Object[] objects = joinPoint.getArgs(); User user = (User) objects[1]; - List modelIds = querySqlReq.getModelIds(); // fetch data permission meta information - ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(modelIds); - List modelSchemaRespList = schemaService.fetchModelSchema(filter, user); - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, modelSchemaRespList); - log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); + SchemaFilterReq filter = new SchemaFilterReq(); + filter.setModelIds(querySqlReq.getModelIds()); + filter.setViewId(querySqlReq.getViewId()); + SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); + List modelIdInView = semanticSchemaResp.getModelResps().stream() + .map(ModelResp::getId).collect(Collectors.toList()); + Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp); + log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege); - Set sensitiveResByModel = getHighSensitiveColsByModelId(modelIds); + Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq); // query user privilege info - AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds, sensitiveResReq); + AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInView, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, modelIds); + Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView); // if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq); @@ -138,7 +138,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource); + return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource); } // if the column has no permission, hit * @@ -147,7 +147,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet); SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); + addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; } @@ -207,21 +207,25 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { Object[] args = point.getArgs(); User user = (User) args[1]; // fetch data permission meta information - List modelIds = queryStructReq.getModelIds(); + SchemaFilterReq filter = new SchemaFilterReq(); + filter.setModelIds(queryStructReq.getModelIds()); + filter.setViewId(queryStructReq.getViewId()); + SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); + List modelIdInView = semanticSchemaResp.getModelResps().stream() + .map(ModelResp::getId).collect(Collectors.toList()); Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq); - log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); + log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege); - Set sensitiveResByModel = getHighSensitiveColsByModelId(modelIds); + Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq); + log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInView, sensitiveResReq); // query user privilege info AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, - modelIds, sensitiveResReq); + modelIdInView, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, - queryStructReq.getModelIds()); + Set resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelIds()); // if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq); @@ -235,7 +239,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource); + return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource); } // if the column has no permission, hit * @@ -243,7 +247,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { .collect(Collectors.toSet()); SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); + addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java index 9ef187d53..38847d341 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java @@ -8,9 +8,6 @@ import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.util.ComponentFactory; import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -19,6 +16,10 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + @Component @Slf4j public class MetaEmbeddingListener implements ApplicationListener { @@ -42,7 +43,8 @@ public class MetaEmbeddingListener implements ApplicationListener { .map(dataItem -> { EmbeddingQuery embeddingQuery = new EmbeddingQuery(); embeddingQuery.setQueryId( - dataItem.getId().toString() + DictWordType.NATURE_SPILT + dataItem.getType().getName()); + dataItem.getId().toString() + DictWordType.NATURE_SPILT + + dataItem.getType().name().toLowerCase()); embeddingQuery.setQuery(dataItem.getName()); Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class); embeddingQuery.setMetadata(meta); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 1ae4d7f57..ee0bc723e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -1,21 +1,20 @@ package com.tencent.supersonic.headless.server.manager; -import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery; -import com.tencent.supersonic.headless.api.pojo.enums.ModelSourceType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; +import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -47,7 +46,6 @@ public class ModelYamlManager { .collect(Collectors.toList())); dataModelYamlTpl.setName(modelResp.getBizName()); dataModelYamlTpl.setSourceId(modelResp.getDatabaseId()); - dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType())); if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) { dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery()); } else { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 7aea80e1d..d35cabff6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -1,12 +1,10 @@ package com.tencent.supersonic.headless.server.manager; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; @@ -31,8 +29,12 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -41,50 +43,31 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Slf4j @Service public class SemanticSchemaManager { - - @Autowired - private LoadingCache loadingCache; private final Catalog catalog; public SemanticSchemaManager(Catalog catalog) { this.catalog = catalog; } - public SemanticModel reload(String rootPath) { + public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) { SemanticModel semanticModel = new SemanticModel(); - semanticModel.setRootPath(rootPath); - Set modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim())) - .collect(Collectors.toSet()); - if (modelIds.isEmpty()) { - log.error("get modelIds empty {}", rootPath); - return semanticModel; - } + semanticModel.setSchemaKey(semanticSchemaResp.getSchemaKey()); Map> dimensionYamlTpls = new HashMap<>(); List dataModelYamlTpls = new ArrayList<>(); List metricYamlTpls = new ArrayList<>(); Map modelIdName = new HashMap<>(); - catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); - DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next()); + catalog.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, + dataModelYamlTpls, metricYamlTpls, modelIdName); + DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp(); semanticModel.setDatabase(DatabaseConverter.convert(databaseResp)); - List modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); - if (!CollectionUtils.isEmpty(modelRelas)) { - semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); + if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) { + semanticModel.setJoinRelations(getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { Map dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource) @@ -104,15 +87,6 @@ public class SemanticSchemaManager { return semanticModel; } - public SemanticModel get(String rootPath) throws Exception { - rootPath = formatKey(rootPath); - SemanticModel schema = loadingCache.get(rootPath); - if (schema == null) { - return null; - } - return schema; - } - public static List getMetrics(final List t) { return getMetricsByMetricYamlTpl(t); } @@ -351,43 +325,4 @@ public class SemanticSchemaManager { metrics.addAll(metricYamlTpls); } - public static String formatKey(String key) { - key = key.trim(); - if (key.startsWith("/")) { - key = key.substring(1); - } - if (key.endsWith("/")) { - key = key.substring(0, key.length() - 1); - } - return key; - } - - @Configuration - @EnableCaching - public class GuavaCacheConfig { - - @Value("${parser.cache.saveMinute:1}") - private Integer saveMinutes = 1; - @Value("${parser.cache.maximumSize:1000}") - private Integer maximumSize = 1000; - - @Bean - public LoadingCache getCache() { - LoadingCache cache - = CacheBuilder.newBuilder() - .expireAfterWrite(saveMinutes, TimeUnit.MINUTES) - .initialCapacity(10) - .maximumSize(maximumSize).build( - new CacheLoader() { - @Override - public SemanticModel load(String key) { - log.info("load SemanticSchema [{}]", key); - return SemanticSchemaManager.this.reload(key); - } - } - ); - return cache; - } - } - } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java similarity index 93% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDO.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java index 332e196d3..9e7f191f2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDO.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java @@ -8,8 +8,8 @@ import lombok.Data; import java.util.Date; @Data -@TableName("s2_view_info") -public class ViewInfoDO { +@TableName("s2_canvas") +public class CanvasDO { /** */ @TableId(type = IdType.AUTO) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java new file mode 100644 index 000000000..ef98e816f --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java @@ -0,0 +1,46 @@ +package com.tencent.supersonic.headless.server.persistence.dataobject; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import java.util.Date; + +@Data +@TableName("s2_view") +public class ViewDO { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long domainId; + + private String name; + + private String bizName; + + private String description; + + private Integer status; + + private String alias; + + private String viewDetail; + + private Date createdAt; + + private String createdBy; + + private Date updatedAt; + + private String updatedBy; + + private String filterSql; + + private String queryConfig; + + private String admin; + + private String adminOrg; + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDOExample.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDOExample.java deleted file mode 100644 index 47df9d744..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDOExample.java +++ /dev/null @@ -1,740 +0,0 @@ -package com.tencent.supersonic.headless.server.persistence.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class ViewInfoDOExample { - /** - * s2_view_info - */ - protected String orderByClause; - - /** - * s2_view_info - */ - protected boolean distinct; - - /** - * s2_view_info - */ - protected List oredCriteria; - - /** - * s2_view_info - */ - protected Integer limitStart; - - /** - * s2_view_info - */ - protected Integer limitEnd; - - /** - * @mbg.generated - */ - public ViewInfoDOExample() { - oredCriteria = new ArrayList(); - } - - /** - * @mbg.generated - */ - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - /** - * @mbg.generated - */ - public String getOrderByClause() { - return orderByClause; - } - - /** - * @mbg.generated - */ - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - /** - * @mbg.generated - */ - public boolean isDistinct() { - return distinct; - } - - /** - * @mbg.generated - */ - public List getOredCriteria() { - return oredCriteria; - } - - /** - * @mbg.generated - */ - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - /** - * @mbg.generated - */ - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - /** - * @mbg.generated - */ - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - /** - * @mbg.generated - */ - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - /** - * @mbg.generated - */ - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - /** - * @mbg.generated - */ - public void setLimitStart(Integer limitStart) { - this.limitStart = limitStart; - } - - /** - * @mbg.generated - */ - public Integer getLimitStart() { - return limitStart; - } - - /** - * @mbg.generated - */ - public void setLimitEnd(Integer limitEnd) { - this.limitEnd = limitEnd; - } - - /** - * @mbg.generated - */ - public Integer getLimitEnd() { - return limitEnd; - } - - /** - * s2_view_info null - */ - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andModelIdIsNull() { - addCriterion("model_id is null"); - return (Criteria) this; - } - - public Criteria andModelIdIsNotNull() { - addCriterion("model_id is not null"); - return (Criteria) this; - } - - public Criteria andModelIdEqualTo(Long value) { - addCriterion("model_id =", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotEqualTo(Long value) { - addCriterion("model_id <>", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThan(Long value) { - addCriterion("model_id >", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThanOrEqualTo(Long value) { - addCriterion("model_id >=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThan(Long value) { - addCriterion("model_id <", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThanOrEqualTo(Long value) { - addCriterion("model_id <=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdIn(List values) { - addCriterion("model_id in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotIn(List values) { - addCriterion("model_id not in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdBetween(Long value1, Long value2) { - addCriterion("model_id between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotBetween(Long value1, Long value2) { - addCriterion("model_id not between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNull() { - addCriterion("created_at is null"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNotNull() { - addCriterion("created_at is not null"); - return (Criteria) this; - } - - public Criteria andCreatedAtEqualTo(Date value) { - addCriterion("created_at =", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotEqualTo(Date value) { - addCriterion("created_at <>", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThan(Date value) { - addCriterion("created_at >", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("created_at >=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThan(Date value) { - addCriterion("created_at <", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThanOrEqualTo(Date value) { - addCriterion("created_at <=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtIn(List values) { - addCriterion("created_at in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotIn(List values) { - addCriterion("created_at not in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtBetween(Date value1, Date value2) { - addCriterion("created_at between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotBetween(Date value1, Date value2) { - addCriterion("created_at not between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNull() { - addCriterion("created_by is null"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNotNull() { - addCriterion("created_by is not null"); - return (Criteria) this; - } - - public Criteria andCreatedByEqualTo(String value) { - addCriterion("created_by =", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotEqualTo(String value) { - addCriterion("created_by <>", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThan(String value) { - addCriterion("created_by >", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThanOrEqualTo(String value) { - addCriterion("created_by >=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThan(String value) { - addCriterion("created_by <", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThanOrEqualTo(String value) { - addCriterion("created_by <=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLike(String value) { - addCriterion("created_by like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotLike(String value) { - addCriterion("created_by not like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByIn(List values) { - addCriterion("created_by in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotIn(List values) { - addCriterion("created_by not in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByBetween(String value1, String value2) { - addCriterion("created_by between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotBetween(String value1, String value2) { - addCriterion("created_by not between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNull() { - addCriterion("updated_at is null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNotNull() { - addCriterion("updated_at is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtEqualTo(Date value) { - addCriterion("updated_at =", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotEqualTo(Date value) { - addCriterion("updated_at <>", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThan(Date value) { - addCriterion("updated_at >", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("updated_at >=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThan(Date value) { - addCriterion("updated_at <", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThanOrEqualTo(Date value) { - addCriterion("updated_at <=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIn(List values) { - addCriterion("updated_at in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotIn(List values) { - addCriterion("updated_at not in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtBetween(Date value1, Date value2) { - addCriterion("updated_at between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotBetween(Date value1, Date value2) { - addCriterion("updated_at not between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNull() { - addCriterion("updated_by is null"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNotNull() { - addCriterion("updated_by is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedByEqualTo(String value) { - addCriterion("updated_by =", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotEqualTo(String value) { - addCriterion("updated_by <>", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThan(String value) { - addCriterion("updated_by >", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThanOrEqualTo(String value) { - addCriterion("updated_by >=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThan(String value) { - addCriterion("updated_by <", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThanOrEqualTo(String value) { - addCriterion("updated_by <=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLike(String value) { - addCriterion("updated_by like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotLike(String value) { - addCriterion("updated_by not like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByIn(List values) { - addCriterion("updated_by in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotIn(List values) { - addCriterion("updated_by not in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByBetween(String value1, String value2) { - addCriterion("updated_by between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotBetween(String value1, String value2) { - addCriterion("updated_by not between", value1, value2, "updatedBy"); - return (Criteria) this; - } - } - - /** - * s2_view_info - */ - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - /** - * s2_view_info null - */ - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewInfoDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java similarity index 74% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewInfoDOMapper.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java index 37dc9ca8d..c453b38e4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewInfoDOMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface ViewInfoDOMapper extends BaseMapper { +public interface CanvasDOMapper extends BaseMapper { } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java new file mode 100644 index 000000000..79be61213 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.headless.server.persistence.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ViewDOMapper extends BaseMapper { + + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/CanvasRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/CanvasRepository.java new file mode 100644 index 000000000..af7272947 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/CanvasRepository.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.server.persistence.repository; + +import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; + +import java.util.List; + +public interface CanvasRepository { + + List getCanvasList(Long domainId); + + CanvasDO getCanvasById(Long id); + + void deleteCanvas(Long id); + + void createCanvas(CanvasDO canvasDO); + + void updateCanvas(CanvasDO canvasDO); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ViewInfoRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ViewInfoRepository.java deleted file mode 100644 index 1fadf340a..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ViewInfoRepository.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.headless.server.persistence.repository; - -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO; - -import java.util.List; - -public interface ViewInfoRepository { - - List getViewInfoList(Long domainId); - - ViewInfoDO getViewInfoById(Long id); - - void deleteViewInfo(Long id); - - void createViewInfo(ViewInfoDO viewInfoDO); - - void updateViewInfo(ViewInfoDO viewInfoDO); -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/CanvasRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/CanvasRepositoryImpl.java new file mode 100644 index 000000000..568b58fc3 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/CanvasRepositoryImpl.java @@ -0,0 +1,48 @@ +package com.tencent.supersonic.headless.server.persistence.repository.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; +import com.tencent.supersonic.headless.server.persistence.mapper.CanvasDOMapper; +import com.tencent.supersonic.headless.server.persistence.repository.CanvasRepository; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class CanvasRepositoryImpl implements CanvasRepository { + + + private CanvasDOMapper canvasDOMapper; + + public CanvasRepositoryImpl(CanvasDOMapper canvasDOMapper) { + this.canvasDOMapper = canvasDOMapper; + } + + @Override + public List getCanvasList(Long domainId) { + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda().eq(CanvasDO::getDomainId, domainId); + return canvasDOMapper.selectList(wrapper); + } + + @Override + public CanvasDO getCanvasById(Long id) { + return canvasDOMapper.selectById(id); + } + + @Override + public void deleteCanvas(Long id) { + canvasDOMapper.deleteById(id); + } + + @Override + public void createCanvas(CanvasDO canvasDO) { + canvasDOMapper.insert(canvasDO); + } + + @Override + public void updateCanvas(CanvasDO canvasDO) { + canvasDOMapper.updateById(canvasDO); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index 568bbef5a..13c9fd223 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -48,8 +48,8 @@ public class StatRepositoryImpl implements StatRepository { statInfos.stream().forEach(stat -> { String dimensions = stat.getDimensions(); String metrics = stat.getMetrics(); - updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.getName(), stat.getModelId()); - updateStatMapInfo(map, metrics, TypeEnums.METRIC.getName(), stat.getModelId()); + updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getModelId()); + updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getModelId()); }); map.forEach((k, v) -> { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ViewInfoRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ViewInfoRepositoryImpl.java deleted file mode 100644 index ac915d2d2..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ViewInfoRepositoryImpl.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.tencent.supersonic.headless.server.persistence.repository.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO; -import com.tencent.supersonic.headless.server.persistence.mapper.ViewInfoDOMapper; -import com.tencent.supersonic.headless.server.persistence.repository.ViewInfoRepository; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public class ViewInfoRepositoryImpl implements ViewInfoRepository { - - - private ViewInfoDOMapper viewInfoDOMapper; - - public ViewInfoRepositoryImpl(ViewInfoDOMapper viewInfoDOMapper) { - this.viewInfoDOMapper = viewInfoDOMapper; - } - - @Override - public List getViewInfoList(Long domainId) { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.lambda().eq(ViewInfoDO::getDomainId, domainId); - return viewInfoDOMapper.selectList(wrapper); - } - - @Override - public ViewInfoDO getViewInfoById(Long id) { - return viewInfoDOMapper.selectById(id); - } - - @Override - public void deleteViewInfo(Long id) { - viewInfoDOMapper.deleteById(id); - } - - @Override - public void createViewInfo(ViewInfoDO viewInfoDO) { - viewInfoDOMapper.insert(viewInfoDO); - } - - @Override - public void updateViewInfo(ViewInfoDO viewInfoDO) { - viewInfoDOMapper.updateById(viewInfoDO); - } - -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java index 3b507f235..91de83646 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java @@ -19,6 +19,10 @@ public class MetaFilter { private List modelIds; + private Long domainId; + + private Long viewId; + private Integer sensitiveLevel; private Integer status; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java new file mode 100644 index 000000000..365cf5661 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java @@ -0,0 +1,57 @@ +package com.tencent.supersonic.headless.server.rest; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.request.CanvasReq; +import com.tencent.supersonic.headless.api.pojo.response.CanvasSchemaResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; +import com.tencent.supersonic.headless.server.service.impl.CanvasServiceImpl; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; + +@RestController +@RequestMapping("/api/semantic/viewInfo") +public class CanvasController { + + private CanvasServiceImpl canvasService; + + public CanvasController(CanvasServiceImpl canvasService) { + this.canvasService = canvasService; + } + + @PostMapping("/createOrUpdateViewInfo") + public CanvasDO createOrUpdateCanvas(@RequestBody CanvasReq canvasReq, HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return canvasService.createOrUpdateCanvas(canvasReq, user); + } + + @GetMapping("/getViewInfoList/{domainId}") + public List getCanvasList(@PathVariable("domainId") Long domainId) { + return canvasService.getCanvasList(domainId); + } + + @DeleteMapping("/deleteViewInfo/{id}") + public void deleteCanvas(@PathVariable("id") Long id) { + canvasService.deleteCanvas(id); + } + + @GetMapping("/getDomainSchemaRela/{domainId}") + public List getDomainSchema(@PathVariable("domainId") Long domainId, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return canvasService.getCanvasSchema(domainId, user); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java index 6aa6fb660..349a6b2f3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java @@ -4,15 +4,15 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.server.service.SchemaService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; @@ -34,11 +34,8 @@ public class SchemaController { private SchemaService schemaService; @PostMapping - public List fetchModelSchema(@RequestBody ModelSchemaFilterReq filter, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return schemaService.fetchModelSchema(filter, user); + public List fetchViewSchema(@RequestBody ViewFilterReq filter) { + return schemaService.fetchViewSchema(filter); } @GetMapping("/domain/list") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java new file mode 100644 index 000000000..2cea09cd6 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java @@ -0,0 +1,64 @@ +package com.tencent.supersonic.headless.server.rest; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.ViewService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; + +@RestController +@RequestMapping("/api/semantic/view") +public class ViewController { + + @Autowired + private ViewService viewService; + + @PostMapping + public ViewResp save(@RequestBody ViewReq viewReq, + HttpServletRequest request, HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return viewService.save(viewReq, user); + } + + @PutMapping + public ViewResp update(@RequestBody ViewReq viewReq, + HttpServletRequest request, HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return viewService.update(viewReq, user); + } + + @GetMapping("/{id}") + public ViewResp getView(@PathVariable("id") Long id) { + return viewService.getView(id); + } + + @GetMapping("/getViewList") + public List getViewList(@RequestParam("domainId") Long domainId) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setDomainId(domainId); + return viewService.getViewList(metaFilter); + } + + @DeleteMapping("/{id}") + public Boolean delete(@PathVariable("id") Long id, + HttpServletRequest request, HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + viewService.delete(id, user); + return true; + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewInfoController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewInfoController.java deleted file mode 100644 index b39816e88..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewInfoController.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.tencent.supersonic.headless.server.rest; - - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.headless.api.pojo.request.ViewInfoReq; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaRelaResp; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO; -import com.tencent.supersonic.headless.server.service.impl.ViewInfoServiceImpl; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.util.List; - -@RestController -@RequestMapping("/api/semantic/viewInfo") -public class ViewInfoController { - - private ViewInfoServiceImpl viewInfoServiceImpl; - - public ViewInfoController(ViewInfoServiceImpl viewInfoServiceImpl) { - this.viewInfoServiceImpl = viewInfoServiceImpl; - } - - @PostMapping("/createOrUpdateViewInfo") - public ViewInfoDO createOrUpdateViewInfo(@RequestBody ViewInfoReq viewInfoReq, HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return viewInfoServiceImpl.createOrUpdateViewInfo(viewInfoReq, user); - } - - @GetMapping("/getViewInfoList/{domainId}") - public List getViewInfoList(@PathVariable("domainId") Long domainId) { - return viewInfoServiceImpl.getViewInfoList(domainId); - } - - @DeleteMapping("/deleteViewInfo/{id}") - public void deleteViewInfo(@PathVariable("id") Long id) { - viewInfoServiceImpl.deleteViewInfo(id); - } - - @GetMapping("/getDomainSchemaRela/{domainId}") - public List getDomainSchema(@PathVariable("domainId") Long domainId, - HttpServletRequest request, - HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - return viewInfoServiceImpl.getDomainSchema(domainId, user); - } - -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java index eb60e6474..104b887b0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java @@ -1,28 +1,25 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ExecutionException; public interface Catalog { - DatabaseResp getDatabase(Long id); - - DatabaseResp getDatabaseByModelId(Long modelId); - DimensionResp getDimension(String bizName, Long modelId); DimensionResp getDimension(Long id); @@ -33,16 +30,17 @@ public interface Catalog { MetricResp getMetric(Long id); - List getModelRela(List modelIds); - - List getModelSchema(List modelIds); - - void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List dataModelYamlTplList, List metricYamlTplList, - Map modelIdName); - ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); List getModelList(List modelIds); + void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp, + Map> dimensionYamlMap, + List dataModelYamlTplList, + List metricYamlTplList, + Map modelIdName); + + SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq); + + List getStatInfo(ItemUseReq itemUseReq) throws ExecutionException; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index b971c7eb6..fe03efb2c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -7,19 +7,13 @@ import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.pojo.ModelFilter; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import java.util.List; import java.util.Map; -import java.util.Set; public interface ModelService { @@ -27,7 +21,7 @@ public interface ModelService { ModelResp updateModel(ModelReq datasourceReq, User user) throws Exception; - List getModelList(ModelFilter modelFilter); + List getModelList(MetaFilter metaFilter); Map getModelMap(); @@ -47,16 +41,8 @@ public interface ModelService { List getModelAdmin(Long id); - ModelSchemaResp fetchSingleModelSchema(Long modelId); - - List fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq); - DatabaseResp getDatabaseByModelId(Long modelId); void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); - void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List dataModelYamlTplList, List metricYamlTplList, - Map modelIdName); - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 2e3b55607..4c34ad624 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -3,21 +3,27 @@ package com.tencent.supersonic.headless.server.service; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; +import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import java.util.List; +import java.util.concurrent.ExecutionException; public interface SchemaService { - List fetchModelSchema(ModelSchemaFilterReq filter, User user); + List fetchViewSchema(ViewFilterReq filter); PageInfo queryDimension(PageDimensionReq pageDimensionReq, User user); @@ -28,4 +34,10 @@ public interface SchemaService { List getDomainList(User user); List getModelList(User user, AuthType authType, Long domainId); + + List getViewList(Long domainId); + + SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq); + + List getStatInfo(ItemUseReq itemUseReq) throws ExecutionException; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java new file mode 100644 index 000000000..871228c9c --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java @@ -0,0 +1,21 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; + +import java.util.List; + +public interface ViewService { + + ViewResp save(ViewReq viewReq, User user); + + ViewResp update(ViewReq viewReq, User user); + + ViewResp getView(Long id); + + List getViewList(MetaFilter metaFilter); + + void delete(Long id, User user); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewInfoServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java similarity index 55% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewInfoServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java index 78c5730cb..c017f13c3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewInfoServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java @@ -4,13 +4,13 @@ package com.tencent.supersonic.headless.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.headless.api.pojo.request.ViewInfoReq; +import com.tencent.supersonic.headless.api.pojo.request.CanvasReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaRelaResp; -import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO; -import com.tencent.supersonic.headless.server.persistence.repository.ViewInfoRepository; +import com.tencent.supersonic.headless.api.pojo.response.CanvasSchemaResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO; +import com.tencent.supersonic.headless.server.persistence.repository.CanvasRepository; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -22,9 +22,9 @@ import java.util.Date; import java.util.List; @Service -public class ViewInfoServiceImpl { +public class CanvasServiceImpl { - private ViewInfoRepository viewInfoRepository; + private CanvasRepository viewInfoRepository; private ModelService modelService; @@ -32,58 +32,58 @@ public class ViewInfoServiceImpl { private MetricService metricService; - public ViewInfoServiceImpl(ViewInfoRepository viewInfoRepository, ModelService modelService, - MetricService metricService, DimensionService dimensionService) { + public CanvasServiceImpl(CanvasRepository viewInfoRepository, ModelService modelService, + MetricService metricService, DimensionService dimensionService) { this.viewInfoRepository = viewInfoRepository; this.dimensionService = dimensionService; this.metricService = metricService; this.modelService = modelService; } - public List getViewInfoList(Long domainId) { - return viewInfoRepository.getViewInfoList(domainId); + public List getCanvasList(Long domainId) { + return viewInfoRepository.getCanvasList(domainId); } - public List getDomainSchema(Long domainId, User user) { - List domainSchemaRelaResps = Lists.newArrayList(); + public List getCanvasSchema(Long domainId, User user) { + List canvasSchemaResps = Lists.newArrayList(); List modelResps = modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); for (ModelResp modelResp : modelResps) { - ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp(); + CanvasSchemaResp canvasSchemaResp = new CanvasSchemaResp(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setModelIds(Lists.newArrayList(modelResp.getId())); List metricResps = metricService.getMetrics(metaFilter); List dimensionResps = dimensionService.getDimensions(metaFilter); - domainSchemaRelaResp.setModel(modelResp); - domainSchemaRelaResp.setDimensions(dimensionResps); - domainSchemaRelaResp.setMetrics(metricResps); - domainSchemaRelaResp.setDomainId(domainId); - domainSchemaRelaResps.add(domainSchemaRelaResp); + canvasSchemaResp.setModel(modelResp); + canvasSchemaResp.setDimensions(dimensionResps); + canvasSchemaResp.setMetrics(metricResps); + canvasSchemaResp.setDomainId(domainId); + canvasSchemaResps.add(canvasSchemaResp); } - return domainSchemaRelaResps; + return canvasSchemaResps; } - public ViewInfoDO createOrUpdateViewInfo(ViewInfoReq viewInfoReq, User user) { - if (viewInfoReq.getId() == null) { - ViewInfoDO viewInfoDO = new ViewInfoDO(); - BeanUtils.copyProperties(viewInfoReq, viewInfoDO); + public CanvasDO createOrUpdateCanvas(CanvasReq canvasReq, User user) { + if (canvasReq.getId() == null) { + CanvasDO viewInfoDO = new CanvasDO(); + BeanUtils.copyProperties(canvasReq, viewInfoDO); viewInfoDO.setCreatedAt(new Date()); viewInfoDO.setCreatedBy(user.getName()); viewInfoDO.setUpdatedAt(new Date()); viewInfoDO.setUpdatedBy(user.getName()); - viewInfoRepository.createViewInfo(viewInfoDO); + viewInfoRepository.createCanvas(viewInfoDO); return viewInfoDO; } - Long id = viewInfoReq.getId(); - ViewInfoDO viewInfoDO = viewInfoRepository.getViewInfoById(id); - BeanUtils.copyProperties(viewInfoReq, viewInfoDO); + Long id = canvasReq.getId(); + CanvasDO viewInfoDO = viewInfoRepository.getCanvasById(id); + BeanUtils.copyProperties(canvasReq, viewInfoDO); viewInfoDO.setUpdatedAt(new Date()); viewInfoDO.setUpdatedBy(user.getName()); - viewInfoRepository.updateViewInfo(viewInfoDO); + viewInfoRepository.updateCanvas(viewInfoDO); return viewInfoDO; } - public void deleteViewInfo(Long id) { - viewInfoRepository.deleteViewInfo(id); + public void deleteCanvas(Long id) { + viewInfoRepository.deleteCanvas(id); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java index cd8838a33..a41b02a48 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java @@ -1,24 +1,31 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; +import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; +import com.tencent.supersonic.headless.server.manager.MetricYamlManager; +import com.tencent.supersonic.headless.server.manager.ModelYamlManager; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.ViewService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Component; @@ -26,7 +33,8 @@ import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; @Slf4j @Component @@ -35,28 +43,21 @@ public class CatalogImpl implements Catalog { private final DatabaseService databaseService; private final ModelService modelService; private final DimensionService dimensionService; - private final ModelService datasourceService; private final MetricService metricService; private final ModelRelaService modelRelaService; + private final ViewService viewService; + private final SchemaService schemaService; - public CatalogImpl(DatabaseService databaseService, - ModelService modelService, DimensionService dimensionService, - ModelService datasourceService, + public CatalogImpl(DatabaseService databaseService, SchemaService schemaService, + ModelService modelService, DimensionService dimensionService, ViewService viewService, MetricService metricService, ModelRelaService modelRelaService) { this.databaseService = databaseService; this.modelService = modelService; this.dimensionService = dimensionService; - this.datasourceService = datasourceService; + this.viewService = viewService; this.metricService = metricService; this.modelRelaService = modelRelaService; - } - - public DatabaseResp getDatabase(Long id) { - return databaseService.getDatabase(id); - } - - public DatabaseResp getDatabaseByModelId(Long modelId) { - return modelService.getDatabaseByModelId(modelId); + this.schemaService = schemaService; } @Override @@ -69,11 +70,6 @@ public class CatalogImpl implements Catalog { return dimensionService.getDimension(id); } - @Override - public List getModelRela(List modelIds) { - return modelRelaService.getModelRela(modelIds); - } - @Override public List getDimensions(MetaFilter metaFilter) { return dimensionService.getDimensions(metaFilter); @@ -91,7 +87,7 @@ public class CatalogImpl implements Catalog { @Override public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) { - return datasourceService.getItemDate(dimension, metric); + return modelService.getItemDate(dimension, metric); } @Override @@ -106,18 +102,43 @@ public class CatalogImpl implements Catalog { } @Override - public List getModelSchema(List modelIds) { - ModelSchemaFilterReq modelSchemaFilterReq = new ModelSchemaFilterReq(); - modelSchemaFilterReq.setModelIds(modelIds); - return modelService.fetchModelSchema(modelSchemaFilterReq); + public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp, + Map> dimensionYamlMap, + List dataModelYamlTplList, + List metricYamlTplList, + Map modelIdName) { + + List modelResps = semanticSchemaResp.getModelResps(); + if (CollectionUtils.isEmpty(modelResps)) { + return; + } + List dimensionResps = semanticSchemaResp.getDimensions(); + Long databaseId = modelResps.get(0).getDatabaseId(); + DatabaseResp databaseResp = databaseService.getDatabase(databaseId); + for (ModelResp modelResp : modelResps) { + modelIdName.put(modelResp.getId(), modelResp.getBizName()); + dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp)); + if (!dimensionYamlMap.containsKey(modelResp.getBizName())) { + dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>()); + } + List dimensionRespList = dimensionResps.stream() + .filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName())) + .collect(Collectors.toList()); + dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml( + dimensionRespList)); + } + List metricResps = new ArrayList<>(semanticSchemaResp.getMetrics()); + metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps)); } @Override - public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List dataModelYamlTplList, List metricYamlTplList, - Map modelIdName) { - datasourceService.getModelYamlTplByModelIds(modelIds, dimensionYamlMap, dataModelYamlTplList, - metricYamlTplList, modelIdName); + public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + return schemaService.fetchSemanticSchema(schemaFilterReq); + } + + @Override + public List getStatInfo(ItemUseReq itemUseReq) throws ExecutionException { + return schemaService.getStatInfo(itemUseReq); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 9c43c7bc3..bc5288adf 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -24,6 +24,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; @@ -32,6 +33,7 @@ import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; @@ -63,6 +65,8 @@ public class DimensionServiceImpl implements DimensionService { private ModelRelaService modelRelaService; + private ViewService viewService; + @Autowired private ApplicationEventPublisher eventPublisher; @@ -71,12 +75,14 @@ public class DimensionServiceImpl implements DimensionService { ModelService modelService, ChatGptHelper chatGptHelper, DatabaseService databaseService, - ModelRelaService modelRelaService) { + ModelRelaService modelRelaService, + ViewService viewService) { this.modelService = modelService; this.dimensionRepository = dimensionRepository; this.chatGptHelper = chatGptHelper; this.databaseService = databaseService; this.modelRelaService = modelRelaService; + this.viewService = viewService; } @Override @@ -94,7 +100,7 @@ public class DimensionServiceImpl implements DimensionService { return; } Long modelId = dimensionReqs.get(0).getModelId(); - List dimensionResps = getDimensionInSameDomain(modelId); + List dimensionResps = getDimensions(modelId); Map bizNameMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); Map nameMap = dimensionResps.stream() @@ -219,6 +225,10 @@ public class DimensionServiceImpl implements DimensionService { if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { return filterByField(dimensionResps, metaFilter.getFieldsDepend()); } + if (metaFilter.getViewId() != null) { + ViewResp viewResp = viewService.getView(metaFilter.getViewId()); + return DimensionConverter.filterByView(dimensionResps, viewResp); + } return dimensionResps; } @@ -317,17 +327,9 @@ public class DimensionServiceImpl implements DimensionService { return dimValueMapsResp; } - private List getDimensionInSameDomain(Long modelId) { - ModelResp modelResp = modelService.getModel(modelId); - Long domainId = modelResp.getDomainId(); - List modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId)); - List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); - return getDimensions(new MetaFilter(modelIds)); - } - private void checkExist(List dimensionReqs) { Long modelId = dimensionReqs.get(0).getModelId(); - List dimensionResps = getDimensionInSameDomain(modelId); + List dimensionResps = getDimensions(modelId); Map bizNameMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); Map nameMap = dimensionResps.stream() diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index 83d3ef7de..d0a62ed61 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -12,21 +12,21 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.RelateDimension; +import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.utils.DataTransformUtils; import com.tencent.supersonic.headless.server.pojo.DataDownload; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DownloadService; -import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.QueryService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -42,6 +42,7 @@ import java.io.OutputStream; import java.net.URLEncoder; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -58,12 +59,16 @@ public class DownloadServiceImpl implements DownloadService { private static final long downloadSize = 10000; - private ModelService modelService; + private MetricService metricService; + + private DimensionService dimensionService; private QueryService queryService; - public DownloadServiceImpl(ModelService modelService, QueryService queryService) { - this.modelService = modelService; + public DownloadServiceImpl(MetricService metricService, + DimensionService dimensionService, QueryService queryService) { + this.metricService = metricService; + this.dimensionService = dimensionService; this.queryService = queryService; } @@ -74,7 +79,7 @@ public class DownloadServiceImpl implements DownloadService { File file = FileUtils.createTmpFile(fileName); try { QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true); - SemanticQueryResp queryResult = (SemanticQueryResp) queryService.queryByReq(querySqlReq, user); + SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData()); } catch (RuntimeException e) { @@ -100,21 +105,30 @@ public class DownloadServiceImpl implements DownloadService { public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception { List metricIds = batchDownloadReq.getMetricIds(); - List modelSchemaRespList = modelService.fetchModelSchema(new ModelSchemaFilterReq()); - Map> metricSchemaMap = getMetricSchemaMap(modelSchemaRespList, metricIds); - Map dimensionRespMap = getDimensionMap(modelSchemaRespList); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(metricIds); + List metricResps = metricService.getMetrics(metaFilter); + Map> metricMap = getMetricMap(metricResps); + List dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension) + .map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream) + .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()); + metaFilter.setIds(dimensionIds); + Map dimensionRespMap = dimensionService.getDimensions(metaFilter) + .stream().collect(Collectors.toMap(DimensionResp::getId, d -> d)); ExcelWriter excelWriter = EasyExcel.write(file).build(); int sheetCount = 1; - for (List metrics : metricSchemaMap.values()) { + for (List metrics : metricMap.values()) { if (CollectionUtils.isEmpty(metrics)) { continue; } - MetricSchemaResp metricSchemaResp = metrics.get(0); - List dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap); - for (MetricSchemaResp metric : metrics) { + MetricResp metricResp = metrics.get(0); + List dimensions = getMetricRelaDimensions(metricResp, dimensionRespMap); + for (MetricResp metric : metrics) { try { - DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq); - SemanticQueryResp queryResult = queryService.queryByReq(downloadStructReq, user); + DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq); + QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq); + querySqlReq.setNeedAuth(true); + SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount) .head(dataDownload.getHeaders()).build(); @@ -219,14 +233,14 @@ public class DownloadServiceImpl implements DownloadService { } } - private DownloadStructReq buildDownloadStructReq(List dimensionResps, MetricResp metricResp, + private DownloadStructReq buildDownloadReq(List dimensionResps, MetricResp metricResp, BatchDownloadReq batchDownloadReq) { DateConf dateConf = batchDownloadReq.getDateInfo(); - Set modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet()); + Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); modelIds.add(metricResp.getModelId()); DownloadStructReq downloadStructReq = new DownloadStructReq(); downloadStructReq.setGroups(dimensionResps.stream() - .map(DimSchemaResp::getBizName).collect(Collectors.toList())); + .map(DimensionResp::getBizName).collect(Collectors.toList())); downloadStructReq.getGroups().add(0, getTimeDimension(dateConf)); Aggregator aggregator = new Aggregator(); aggregator.setColumn(metricResp.getBizName()); @@ -248,31 +262,27 @@ public class DownloadServiceImpl implements DownloadService { } } - private Map> getMetricSchemaMap(List modelSchemaRespList, - List metricIds) { - return modelSchemaRespList.stream().flatMap(modelSchemaResp - -> modelSchemaResp.getMetrics().stream()) - .filter(metricSchemaResp -> metricIds.contains(metricSchemaResp.getId())) - .collect(Collectors.groupingBy(MetricSchemaResp::getRelaDimensionIdKey)); - } - - private Map getDimensionMap(List modelSchemaRespList) { - return modelSchemaRespList.stream().flatMap(modelSchemaResp - -> modelSchemaResp.getDimensions().stream()) - .collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp)); + private Map> getMetricMap(List metricResps) { + for (MetricResp metricResp : metricResps) { + List drillDownDimensions = metricService.getDrillDownDimension(metricResp.getId()); + RelateDimension relateDimension = RelateDimension.builder() + .drillDownDimensions(drillDownDimensions).build(); + metricResp.setRelateDimension(relateDimension); + } + return metricResps.stream().collect(Collectors.groupingBy(MetricResp::getRelaDimensionIdKey)); } private Map getDimensionNameMap(List queryColumns) { return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn)); } - private List getMetricRelaDimensions(MetricSchemaResp metricSchemaResp, - Map dimensionRespMap) { - if (metricSchemaResp.getRelateDimension() == null - || CollectionUtils.isEmpty(metricSchemaResp.getRelateDimension().getDrillDownDimensions())) { + private List getMetricRelaDimensions(MetricResp metricResp, + Map dimensionRespMap) { + if (metricResp.getRelateDimension() == null + || CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { return Lists.newArrayList(); } - return metricSchemaResp.getRelateDimension().getDrillDownDimensions() + return metricResp.getRelateDimension().getDrillDownDimensions() .stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId())) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index a1148a8a3..794f13fc3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -25,6 +25,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; @@ -35,6 +36,7 @@ import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; import lombok.extern.slf4j.Slf4j; @@ -65,6 +67,8 @@ public class MetricServiceImpl implements MetricService { private CollectService collectService; + private ViewService viewService; + private ApplicationEventPublisher eventPublisher; public MetricServiceImpl(MetricRepository metricRepository, @@ -72,6 +76,7 @@ public class MetricServiceImpl implements MetricService { DomainService domainService, ChatGptHelper chatGptHelper, CollectService collectService, + ViewService viewService, ApplicationEventPublisher eventPublisher) { this.domainService = domainService; this.metricRepository = metricRepository; @@ -79,6 +84,7 @@ public class MetricServiceImpl implements MetricService { this.chatGptHelper = chatGptHelper; this.eventPublisher = eventPublisher; this.collectService = collectService; + this.viewService = viewService; } @Override @@ -98,7 +104,7 @@ public class MetricServiceImpl implements MetricService { return; } Long modelId = metricReqs.get(0).getModelId(); - List metricResps = getMetricInSameDomain(modelId); + List metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId))); Map bizNameMap = metricResps.stream() .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); Map nameMap = metricResps.stream() @@ -186,7 +192,11 @@ public class MetricServiceImpl implements MetricService { List collectList = collectService.getCollectList(user.getName()); List collectIds = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList()); if (pageMetricReq.isHasCollect()) { - metricFilter.setIds(collectIds); + if (CollectionUtils.isEmpty(collectIds)) { + metricFilter.setIds(Lists.newArrayList(-1L)); + } else { + metricFilter.setIds(collectIds); + } } PageInfo metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(), pageMetricReq.getPageSize()) @@ -211,6 +221,10 @@ public class MetricServiceImpl implements MetricService { if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { return filterByField(metricResps, metaFilter.getFieldsDepend()); } + if (metaFilter.getViewId() != null) { + ViewResp viewResp = viewService.getView(metaFilter.getViewId()); + return MetricConverter.filterByView(metricResps, viewResp); + } return metricResps; } @@ -321,16 +335,27 @@ public class MetricServiceImpl implements MetricService { @Override public List getDrillDownDimension(Long metricId) { + List drillDownDimensions = Lists.newArrayList(); MetricResp metricResp = getMetric(metricId); if (metricResp == null) { - return Lists.newArrayList(); + return drillDownDimensions; } if (metricResp.getRelateDimension() != null && !CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { - return metricResp.getRelateDimension().getDrillDownDimensions(); + drillDownDimensions.addAll(metricResp.getRelateDimension().getDrillDownDimensions()); } ModelResp modelResp = modelService.getModel(metricResp.getModelId()); - return modelResp.getDrillDownDimensions(); + if (modelResp.getDrillDownDimensions() == null) { + return drillDownDimensions; + } + for (DrillDownDimension drillDownDimension : modelResp.getDrillDownDimensions()) { + if (!drillDownDimensions.stream().map(DrillDownDimension::getDimensionId) + .collect(Collectors.toList()).contains(drillDownDimension.getDimensionId())) { + drillDownDimension.setInheritedFromModel(true); + drillDownDimensions.add(drillDownDimension); + } + } + return drillDownDimensions; } @Override @@ -361,7 +386,9 @@ public class MetricServiceImpl implements MetricService { private void checkExist(List metricReqs) { Long modelId = metricReqs.get(0).getModelId(); - List metricResps = getMetricInSameDomain(modelId); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); + List metricResps = getMetrics(metaFilter); Map bizNameMap = metricResps.stream() .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); Map nameMap = metricResps.stream() @@ -384,14 +411,6 @@ public class MetricServiceImpl implements MetricService { } } - private List getMetricInSameDomain(Long modelId) { - ModelResp modelResp = modelService.getModel(modelId); - Long domainId = modelResp.getDomainId(); - List modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId)); - List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); - return getMetrics(new MetaFilter(modelIds)); - } - private List convertList(List metricDOS) { return convertList(metricDOS, Lists.newArrayList()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index 50aad4483..ae2398770 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -4,7 +4,6 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; @@ -14,29 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.request.DateInfoReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; -import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; -import com.tencent.supersonic.headless.server.manager.MetricYamlManager; -import com.tencent.supersonic.headless.server.manager.ModelYamlManager; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; @@ -49,6 +38,7 @@ import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.ModelConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; @@ -67,7 +57,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -90,6 +79,8 @@ public class ModelServiceImpl implements ModelService { private UserService userService; + private ViewService viewService; + private DateInfoRepository dateInfoRepository; public ModelServiceImpl(ModelRepository modelRepository, @@ -99,6 +90,7 @@ public class ModelServiceImpl implements ModelService { ModelRelaService modelRelaService, DomainService domainService, UserService userService, + ViewService viewService, DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; this.databaseService = databaseService; @@ -107,6 +99,7 @@ public class ModelServiceImpl implements ModelService { this.domainService = domainService; this.modelRelaService = modelRelaService; this.userService = userService; + this.viewService = viewService; this.dateInfoRepository = dateInfoRepository; } @@ -153,8 +146,16 @@ public class ModelServiceImpl implements ModelService { } @Override - public List getModelList(ModelFilter modelFilter) { - return ModelConverter.convertList(modelRepository.getModelList(modelFilter)); + public List getModelList(MetaFilter metaFilter) { + ModelFilter modelFilter = new ModelFilter(); + BeanUtils.copyProperties(metaFilter, modelFilter); + List modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter)); + if (modelFilter.getViewId() != null) { + ViewResp viewResp = viewService.getView(modelFilter.getViewId()); + return modelResps.stream().filter(modelResp -> viewResp.getAllModels().contains(modelResp.getId())) + .collect(Collectors.toList()); + } + return modelResps; } @Override @@ -383,52 +384,6 @@ public class ModelServiceImpl implements ModelService { return Lists.newArrayList(); } - @Override - public ModelSchemaResp fetchSingleModelSchema(Long modelId) { - ModelResp modelResp = getModel(modelId); - ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); - BeanUtils.copyProperties(modelResp, modelSchemaResp); - modelSchemaResp.setDimensions(generateDimSchema(modelId)); - modelSchemaResp.setMetrics(generateMetricSchema(modelId, modelResp)); - return modelSchemaResp; - } - - @Override - public List fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq) { - List modelSchemaRespList = new ArrayList<>(); - List modelIds = modelSchemaFilterReq.getModelIds(); - if (CollectionUtils.isEmpty(modelIds)) { - modelIds = generateModelIdsReq(modelSchemaFilterReq); - } - MetaFilter metaFilter = new MetaFilter(modelIds); - metaFilter.setStatus(StatusEnum.ONLINE.getCode()); - Map> metricRespMap = metricService.getMetrics(metaFilter) - .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionRespsMap = dimensionService.getDimensions(metaFilter) - .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); - List modelRelas = modelRelaService.getModelRela(modelIds); - for (Long modelId : modelIds) { - ModelResp modelResp = getModelMap().get(modelId); - if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) { - continue; - } - List metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList()); - List metricSchemaResps = metricResps.stream().map(metricResp -> - convert(metricResp, modelResp)).collect(Collectors.toList()); - List dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()) - .stream().map(this::convert).collect(Collectors.toList()); - ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); - BeanUtils.copyProperties(modelResp, modelSchemaResp); - modelSchemaResp.setDimensions(dimensionResps); - modelSchemaResp.setMetrics(metricSchemaResps); - modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela - -> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId)) - .collect(Collectors.toList())); - modelSchemaRespList.add(modelSchemaResp); - } - return modelSchemaRespList; - } - @Override public DatabaseResp getDatabaseByModelId(Long modelId) { ModelResp modelResp = getModel(modelId); @@ -464,37 +419,6 @@ public class ModelServiceImpl implements ModelService { return modelRepository.getModelById(id); } - private List generateMetricSchema(Long modelId, ModelResp modelResp) { - List metricSchemaDescList = new ArrayList<>(); - List metricResps = metricService.getMetrics(new MetaFilter(Lists.newArrayList(modelId))); - metricResps.forEach(metricResp -> metricSchemaDescList.add(convert(metricResp, modelResp))); - return metricSchemaDescList; - } - - private List generateDimSchema(Long modelId) { - List dimDescList = dimensionService.getDimensions(new MetaFilter(Lists.newArrayList(modelId))); - return dimDescList.stream().map(this::convert).collect(Collectors.toList()); - } - - private DimSchemaResp convert(DimensionResp dimensionResp) { - DimSchemaResp dimSchemaResp = new DimSchemaResp(); - BeanUtils.copyProperties(dimensionResp, dimSchemaResp); - dimSchemaResp.setUseCnt(0L); - return dimSchemaResp; - } - - private MetricSchemaResp convert(MetricResp metricResp, ModelResp modelResp) { - MetricSchemaResp metricSchemaResp = new MetricSchemaResp(); - BeanUtils.copyProperties(metricResp, metricSchemaResp); - RelateDimension relateDimension = metricResp.getRelateDimension(); - if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { - metricSchemaResp.setRelateDimension(RelateDimension.builder() - .drillDownDimensions(modelResp.getDrillDownDimensions()).build()); - } - metricSchemaResp.setUseCnt(0L); - return metricSchemaResp; - } - private List convert(List dateInfoDOList) { List dateInfoCommendList = new ArrayList<>(); dateInfoDOList.forEach(dateInfoDO -> { @@ -506,13 +430,6 @@ public class ModelServiceImpl implements ModelService { return dateInfoCommendList; } - private List generateModelIdsReq(ModelSchemaFilterReq filter) { - if (Objects.nonNull(filter) && !CollectionUtils.isEmpty(filter.getModelIds())) { - return filter.getModelIds(); - } - return new ArrayList<>(getModelMap().keySet()); - } - public static boolean checkAdminPermission(Set orgIds, User user, ModelResp modelResp) { List admins = modelResp.getAdmins(); List adminOrgs = modelResp.getAdminOrgs(); @@ -561,30 +478,4 @@ public class ModelServiceImpl implements ModelService { return false; } - @Override - public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List dataModelYamlTplList, List metricYamlTplList, - Map modelIdName) { - for (Long modelId : modelIds) { - ModelResp modelResp = getModel(modelId); - modelIdName.put(modelId, modelResp.getBizName()); - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); - List metricResps = metricService.getMetrics(metaFilter); - metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps)); - Long databaseId = modelResp.getDatabaseId(); - DatabaseResp databaseResp = databaseService.getDatabase(databaseId); - List dimensionResps = dimensionService.getDimensions(metaFilter); - - dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp)); - if (!dimensionYamlMap.containsKey(modelResp.getBizName())) { - dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>()); - } - List dimensionRespList = dimensionResps.stream() - .filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName())) - .collect(Collectors.toList()); - dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml( - dimensionRespList)); - } - } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 68eb93336..f09b2db31 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.headless.server.service.impl; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -12,19 +10,17 @@ import com.tencent.supersonic.common.pojo.enums.ApiItemType; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Item; import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; @@ -33,8 +29,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.parser.DefaultQueryParser; @@ -49,32 +45,27 @@ import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.QueryService; -import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.servlet.http.HttpServletRequest; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Service @Slf4j public class QueryServiceImpl implements QueryService { - - protected final Cache> itemUseCache = - CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); - - private final StatUtils statUtils; + private StatUtils statUtils; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; private final Catalog catalog; @@ -124,7 +115,7 @@ public class QueryServiceImpl implements QueryService { } StatUtils.get().setUseResultCache(false); //3 query - QueryStatement queryStatement = buildQueryStatement(queryReq, user); + QueryStatement queryStatement = buildQueryStatement(queryReq); SemanticQueryResp result = query(queryStatement); //4 reset cache and set stateInfo Boolean setCacheSuccess = queryCache.put(cacheKey, result); @@ -145,49 +136,51 @@ public class QueryServiceImpl implements QueryService { } } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception { - ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(querySqlReq.getModelIds()); - SchemaService schemaService = ContextUtils.getBean(SchemaService.class); - List modelSchemaResps = schemaService.fetchModelSchema(filter, user); - QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, modelSchemaResps); + private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq) throws Exception { + SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq); + SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); + QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, semanticSchemaResp); queryStatement.setModelIds(querySqlReq.getModelIds()); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - SemanticModel semanticModel = semanticSchemaManager.get(querySqlReq.getModelIdStr()); + queryStatement.setSemanticSchemaResp(semanticSchemaResp); + SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp); queryStatement.setSemanticModel(semanticModel); return queryStatement; } - private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception { + private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq) throws Exception { if (semanticQueryReq instanceof QuerySqlReq) { - return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user); + return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq); } if (semanticQueryReq instanceof QueryStructReq) { - return buildStructQueryStatement((QueryStructReq) semanticQueryReq, user); + return buildStructQueryStatement((QueryStructReq) semanticQueryReq); } if (semanticQueryReq instanceof QueryMultiStructReq) { - return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user); + return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); } return null; } - private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq, User user) throws Exception { + private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) throws Exception { + SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq); + SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructReq); queryStatement.setIsS2SQL(false); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); - SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr()); + queryStatement.setViewId(queryStatement.getQueryStructReq().getViewId()); + queryStatement.setSemanticSchemaResp(semanticSchemaResp); + SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp); queryStatement.setSemanticModel(semanticModel); return queryStatement; } - private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq, User user) + private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq) throws Exception { List sqlParsers = new ArrayList<>(); for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - QueryStatement queryStatement = buildQueryStatement(queryStructReq, user); - SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr()); + QueryStatement queryStatement = buildQueryStatement(queryStructReq); + SemanticModel semanticModel = queryStatement.getSemanticModel(); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); queryStatement.setSemanticModel(semanticModel); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); @@ -198,6 +191,13 @@ public class QueryServiceImpl implements QueryService { return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); } + private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { + SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); + schemaFilterReq.setViewId(semanticQueryReq.getViewId()); + schemaFilterReq.setModelIds(semanticQueryReq.getModelIds()); + return schemaFilterReq; + } + @Override @SneakyThrows public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { @@ -208,20 +208,13 @@ public class QueryServiceImpl implements QueryService { @Override @SneakyThrows public List getStatInfo(ItemUseReq itemUseReq) { - if (itemUseReq.getCacheEnable()) { - return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> { - List data = statUtils.getStatInfo(itemUseReq); - itemUseCache.put(JsonUtil.toString(itemUseReq), data); - return data; - }); - } - return statUtils.getStatInfo(itemUseReq); + return catalog.getStatInfo(itemUseReq); } @Override public ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception { T queryReq = explainSqlReq.getQueryReq(); - QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq, user); + QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq); queryStatement = plan(queryStatement); return getExplainResp(queryStatement); } @@ -297,7 +290,7 @@ public class QueryServiceImpl implements QueryService { } private void authCheck(AppDetailResp appDetailResp, List ids, ApiItemType type) { - Set idsInApp = appDetailResp.getConfig().getAllItems().stream() + Set idsInApp = appDetailResp.allItems().stream() .filter(item -> type.equals(item.getType())).map(Item::getId).collect(Collectors.toSet()); if (!idsInApp.containsAll(ids)) { throw new InvalidArgumentException("查询范围超过应用申请范围, 请检查"); @@ -346,9 +339,7 @@ public class QueryServiceImpl implements QueryService { //3 execute if (queryExecutor != null) { semanticQueryResp = queryExecutor.execute(queryStatement); - if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { - queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds()); - } + queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getSemanticSchemaResp()); } return semanticQueryResp; } catch (Exception e) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index e07708282..80bfddecf 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -1,15 +1,24 @@ package com.tencent.supersonic.headless.server.service.impl; import com.github.pagehelper.PageInfo; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.pojo.enums.SchemaType; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; -import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; +import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq; +import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; @@ -18,19 +27,33 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.ViewService; +import com.tencent.supersonic.headless.server.utils.DimensionConverter; +import com.tencent.supersonic.headless.server.utils.MetricConverter; +import com.tencent.supersonic.headless.server.utils.StatUtils; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; @@ -39,37 +62,115 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; @Service public class SchemaServiceImpl implements SchemaService { - private final QueryService queryService; + protected final Cache> itemUseCache = + CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); + private final StatUtils statUtils; private final ModelService modelService; private final DimensionService dimensionService; private final MetricService metricService; private final DomainService domainService; + private final ViewService viewService; + private final ModelRelaService modelRelaService; - public SchemaServiceImpl(QueryService queryService, - ModelService modelService, + public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService, MetricService metricService, - DomainService domainService) { - this.queryService = queryService; + DomainService domainService, + ViewService viewService, + ModelRelaService modelRelaService, + StatUtils statUtils) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; this.domainService = domainService; + this.viewService = viewService; + this.modelRelaService = modelRelaService; + this.statUtils = statUtils; } + @SneakyThrows @Override - public List fetchModelSchema(ModelSchemaFilterReq filter, User user) { - List domainSchemaDescList = modelService.fetchModelSchema(filter); - ItemUseReq itemUseCommend = new ItemUseReq(); - itemUseCommend.setModelIds(filter.getModelIds()); - - List statInfos = queryService.getStatInfo(itemUseCommend); - log.debug("statInfos:{}", statInfos); - fillCnt(domainSchemaDescList, statInfos); - return domainSchemaDescList; + public List fetchViewSchema(ViewFilterReq filter) { + List viewSchemaResps = new ArrayList<>(); + List viewIds = filter.getViewIds(); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setStatus(StatusEnum.ONLINE.getCode()); + metaFilter.setIds(viewIds); + List viewResps = viewService.getViewList(metaFilter); + List modelIds = viewResps.stream().map(ViewResp::getAllModels) + .flatMap(Collection::stream).collect(Collectors.toList()); + metaFilter.setModelIds(modelIds); + metaFilter.setIds(Lists.newArrayList()); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); + metaFilter.setIds(modelIds); + List modelResps = modelService.getModelList(metaFilter); + Map viewRespMap = getViewMap(viewResps); + for (Long viewId : viewRespMap.keySet()) { + ViewResp viewResp = viewRespMap.get(viewId); + if (viewResp == null || !StatusEnum.ONLINE.getCode().equals(viewResp.getStatus())) { + continue; + } + List metricSchemaResps = MetricConverter.filterByView(metricResps, viewResp) + .stream().map(this::convert).collect(Collectors.toList()); + List dimSchemaResps = DimensionConverter.filterByView(dimensionResps, viewResp) + .stream().map(this::convert).collect(Collectors.toList()); + ViewSchemaResp viewSchemaResp = new ViewSchemaResp(); + BeanUtils.copyProperties(viewResp, viewSchemaResp); + viewSchemaResp.setDimensions(dimSchemaResps); + viewSchemaResp.setMetrics(metricSchemaResps); + viewSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> + viewResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); + viewSchemaResps.add(viewSchemaResp); + } + fillStaticInfo(viewSchemaResps); + return viewSchemaResps; } - private void fillCnt(List domainSchemaDescList, List statInfos) { + public ViewSchemaResp fetchViewSchema(Long viewId) { + if (viewId == null) { + return null; + } + return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); + } + + public List fetchModelSchemaResps(List modelIds) { + List modelSchemaResps = Lists.newArrayList(); + if (CollectionUtils.isEmpty(modelIds)) { + return modelSchemaResps; + } + MetaFilter metaFilter = new MetaFilter(modelIds); + metaFilter.setStatus(StatusEnum.ONLINE.getCode()); + Map> metricRespMap = metricService.getMetrics(metaFilter) + .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); + Map> dimensionRespsMap = dimensionService.getDimensions(metaFilter) + .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); + List modelRelas = modelRelaService.getModelRela(modelIds); + Map modelMap = modelService.getModelMap(); + for (Long modelId : modelIds) { + ModelResp modelResp = modelMap.get(modelId); + if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) { + continue; + } + List metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList()); + List metricSchemaResps = metricResps.stream() + .map(this::convert).collect(Collectors.toList()); + List dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()) + .stream().map(this::convert).collect(Collectors.toList()); + ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); + BeanUtils.copyProperties(modelResp, modelSchemaResp); + modelSchemaResp.setDimensions(dimensionResps); + modelSchemaResp.setMetrics(metricSchemaResps); + modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela + -> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId)) + .collect(Collectors.toList())); + modelSchemaResps.add(modelSchemaResp); + } + return modelSchemaResps; + + } + + private void fillCnt(List viewSchemaResps, List statInfos) { Map typeIdAndStatPair = statInfos.stream() .collect(Collectors.toMap( @@ -77,43 +178,45 @@ public class SchemaServiceImpl implements SchemaService { itemUseInfo -> itemUseInfo, (item1, item2) -> item1)); log.debug("typeIdAndStatPair:{}", typeIdAndStatPair); - for (ModelSchemaResp domainSchemaDesc : domainSchemaDescList) { - fillDimCnt(domainSchemaDesc, typeIdAndStatPair); - fillMetricCnt(domainSchemaDesc, typeIdAndStatPair); + for (ViewSchemaResp viewSchemaResp : viewSchemaResps) { + fillDimCnt(viewSchemaResp, typeIdAndStatPair); + fillMetricCnt(viewSchemaResp, typeIdAndStatPair); } } - private void fillMetricCnt(ModelSchemaResp domainSchemaDesc, Map typeIdAndStatPair) { - List metrics = domainSchemaDesc.getMetrics(); - if (CollectionUtils.isEmpty(domainSchemaDesc.getMetrics())) { + private void fillMetricCnt(ViewSchemaResp viewSchemaResp, Map typeIdAndStatPair) { + List metrics = viewSchemaResp.getMetrics(); + if (CollectionUtils.isEmpty(viewSchemaResp.getMetrics())) { return; } if (!CollectionUtils.isEmpty(metrics)) { metrics.stream().forEach(metric -> { - String key = TypeEnums.METRIC.getName() + AT_SYMBOL + AT_SYMBOL + metric.getBizName(); + String key = TypeEnums.METRIC.name().toLowerCase() + + AT_SYMBOL + AT_SYMBOL + metric.getBizName(); if (typeIdAndStatPair.containsKey(key)) { metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); } }); } - domainSchemaDesc.setMetrics(metrics); + viewSchemaResp.setMetrics(metrics); } - private void fillDimCnt(ModelSchemaResp domainSchemaDesc, Map typeIdAndStatPair) { - List dimensions = domainSchemaDesc.getDimensions(); - if (CollectionUtils.isEmpty(domainSchemaDesc.getDimensions())) { + private void fillDimCnt(ViewSchemaResp viewSchemaResp, Map typeIdAndStatPair) { + List dimensions = viewSchemaResp.getDimensions(); + if (CollectionUtils.isEmpty(viewSchemaResp.getDimensions())) { return; } if (!CollectionUtils.isEmpty(dimensions)) { dimensions.stream().forEach(dim -> { - String key = TypeEnums.DIMENSION.getName() + AT_SYMBOL + AT_SYMBOL + dim.getBizName(); + String key = TypeEnums.DIMENSION.name().toLowerCase() + + AT_SYMBOL + AT_SYMBOL + dim.getBizName(); if (typeIdAndStatPair.containsKey(key)) { dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt()); } }); } - domainSchemaDesc.setDimensions(dimensions); + viewSchemaResp.setDimensions(dimensions); } @Override @@ -135,7 +238,7 @@ public class SchemaServiceImpl implements SchemaService { } else if (TypeEnums.DIMENSION.equals(schemaItemQueryReq.getType())) { return dimensionService.getDimensions(metaFilter); } - throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().getName()); + throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().name()); } @Override @@ -148,4 +251,97 @@ public class SchemaServiceImpl implements SchemaService { return modelService.getModelListWithAuth(user, domainId, authTypeEnum); } + @Override + public List getViewList(Long domainId) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setDomainId(domainId); + return viewService.getViewList(metaFilter); + } + + @Override + public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); + semanticSchemaResp.setViewId(schemaFilterReq.getViewId()); + semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds()); + if (schemaFilterReq.getViewId() != null) { + ViewSchemaResp viewSchemaResp = fetchViewSchema(schemaFilterReq.getViewId()); + BeanUtils.copyProperties(viewSchemaResp, semanticSchemaResp); + List modelIds = viewSchemaResp.getAllModels(); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(modelIds); + List modelList = modelService.getModelList(metaFilter); + metaFilter.setModelIds(modelIds); + List modelRelas = modelRelaService.getModelRela(modelIds); + semanticSchemaResp.setModelResps(modelList); + semanticSchemaResp.setModelRelas(modelRelas); + semanticSchemaResp.setModelIds(modelIds); + semanticSchemaResp.setSchemaType(SchemaType.VIEW); + } else if (!CollectionUtils.isEmpty(schemaFilterReq.getModelIds())) { + List modelSchemaResps = fetchModelSchemaResps(schemaFilterReq.getModelIds()); + semanticSchemaResp.setMetrics(modelSchemaResps.stream().map(ModelSchemaResp::getMetrics) + .flatMap(Collection::stream).collect(Collectors.toList())); + semanticSchemaResp.setDimensions(modelSchemaResps.stream().map(ModelSchemaResp::getDimensions) + .flatMap(Collection::stream).collect(Collectors.toList())); + semanticSchemaResp.setModelRelas(modelSchemaResps.stream().map(ModelSchemaResp::getModelRelas) + .flatMap(Collection::stream).collect(Collectors.toList())); + semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); + semanticSchemaResp.setSchemaType(SchemaType.MODEL); + } + if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { + DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); + semanticSchemaResp.setDatabaseResp(databaseResp); + } + return semanticSchemaResp; + } + + @SneakyThrows + @Override + public List getStatInfo(ItemUseReq itemUseReq) { + if (itemUseReq.getCacheEnable()) { + return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> { + List data = statUtils.getStatInfo(itemUseReq); + itemUseCache.put(JsonUtil.toString(itemUseReq), data); + return data; + }); + } + return statUtils.getStatInfo(itemUseReq); + } + + private void fillStaticInfo(List viewSchemaResps) throws ExecutionException { + List viewIds = viewSchemaResps.stream() + .map(ViewSchemaResp::getId).collect(Collectors.toList()); + ItemUseReq itemUseReq = new ItemUseReq(); + itemUseReq.setModelIds(viewIds); + + List statInfos = getStatInfo(itemUseReq); + log.debug("statInfos:{}", statInfos); + fillCnt(viewSchemaResps, statInfos); + } + + private Map getViewMap(List viewResps) { + if (CollectionUtils.isEmpty(viewResps)) { + return new HashMap<>(); + } + return viewResps.stream().collect( + Collectors.toMap(ViewResp::getId, viewResp -> viewResp)); + } + + private DimSchemaResp convert(DimensionResp dimensionResp) { + DimSchemaResp dimSchemaResp = new DimSchemaResp(); + BeanUtils.copyProperties(dimensionResp, dimSchemaResp); + return dimSchemaResp; + } + + private MetricSchemaResp convert(MetricResp metricResp) { + MetricSchemaResp metricSchemaResp = new MetricSchemaResp(); + BeanUtils.copyProperties(metricResp, metricSchemaResp); + return metricSchemaResp; + } + + private ModelResp convert(ModelSchemaResp modelSchemaResp) { + ModelResp modelResp = new ModelResp(); + BeanUtils.copyProperties(modelSchemaResp, modelResp); + return modelResp; + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java new file mode 100644 index 000000000..be083dec3 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java @@ -0,0 +1,100 @@ +package com.tencent.supersonic.headless.server.service.impl; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.google.common.collect.Lists; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; +import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.ViewService; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class ViewServiceImpl + extends ServiceImpl implements ViewService { + + @Override + public ViewResp save(ViewReq viewReq, User user) { + viewReq.createdBy(user.getName()); + ViewDO viewDO = convert(viewReq); + viewDO.setStatus(StatusEnum.ONLINE.getCode()); + save(viewDO); + return convert(viewDO); + } + + @Override + public ViewResp update(ViewReq viewReq, User user) { + viewReq.updatedBy(user.getName()); + ViewDO viewDO = convert(viewReq); + updateById(viewDO); + return convert(viewDO); + } + + @Override + public ViewResp getView(Long id) { + ViewDO viewDO = getById(id); + return convert(viewDO); + } + + @Override + public List getViewList(MetaFilter metaFilter) { + QueryWrapper wrapper = new QueryWrapper<>(); + if (metaFilter.getDomainId() != null) { + wrapper.lambda().eq(ViewDO::getDomainId, metaFilter.getDomainId()); + } + if (!CollectionUtils.isEmpty(metaFilter.getIds())) { + wrapper.lambda().in(ViewDO::getId, metaFilter.getIds()); + } + wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode()); + return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); + } + + @Override + public void delete(Long id, User user) { + ViewDO viewDO = getById(id); + viewDO.setStatus(StatusEnum.DELETED.getCode()); + viewDO.setUpdatedBy(user.getName()); + viewDO.setUpdatedAt(new Date()); + updateById(viewDO); + } + + private ViewResp convert(ViewDO viewDO) { + ViewResp viewResp = new ViewResp(); + BeanMapper.mapper(viewDO, viewResp); + viewResp.setViewDetail(JSONObject.parseObject(viewDO.getViewDetail(), ViewDetail.class)); + if (viewDO.getQueryConfig() != null) { + viewResp.setQueryConfig(JSONObject.parseObject(viewDO.getQueryConfig(), QueryConfig.class)); + } + viewResp.setAdmins(StringUtils.isBlank(viewDO.getAdmin()) + ? Lists.newArrayList() : Arrays.asList(viewDO.getAdmin().split(","))); + viewResp.setAdminOrgs(StringUtils.isBlank(viewDO.getAdminOrg()) + ? Lists.newArrayList() : Arrays.asList(viewDO.getAdminOrg().split(","))); + viewResp.setTypeEnum(TypeEnums.VIEW); + return viewResp; + } + + private ViewDO convert(ViewReq viewReq) { + ViewDO viewDO = new ViewDO(); + BeanMapper.mapper(viewReq, viewDO); + viewDO.setViewDetail(JSONObject.toJSONString(viewReq.getViewDetail())); + viewDO.setQueryConfig(JSONObject.toJSONString(viewReq.getQueryConfig())); + return viewDO; + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index fd79b54ba..72aa612a0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; @@ -19,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class DimensionConverter { @@ -77,4 +79,11 @@ public class DimensionConverter { return dimensionResp; } + public static List filterByView(List dimensionResps, ViewResp viewResp) { + return dimensionResps.stream().filter(dimensionResp -> + viewResp.getAllDimensions().contains(dimensionResp.getId()) + || viewResp.getAllIncludeAllModels().contains(dimensionResp.getModelId())) + .collect(Collectors.toList()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 5680c9ade..282b6ff39 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -14,12 +14,14 @@ import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import org.springframework.beans.BeanUtils; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class MetricConverter { @@ -96,4 +98,11 @@ public class MetricConverter { return metricResp; } + public static List filterByView(List metricResps, ViewResp viewResp) { + return metricResps.stream().filter(metricResp -> + viewResp.getAllMetrics().contains(metricResp.getId()) + || viewResp.getAllIncludeAllModels().contains(metricResp.getModelId())) + .collect(Collectors.toList()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index ae6e88681..acba99c5c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -9,25 +9,33 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.Catalog; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -37,14 +45,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; @Component @Slf4j @@ -59,36 +59,31 @@ public class QueryReqConverter { @Autowired private SqlGenerateUtils sqlGenerateUtils; - @Autowired - private Catalog catalog; + public QueryStatement convert(QuerySqlReq querySQLReq, + SemanticSchemaResp semanticSchemaResp) throws Exception { - public QueryStatement convert(QuerySqlReq querySqlReq, - List modelSchemaResps) throws Exception { - - if (CollectionUtils.isEmpty(modelSchemaResps)) { + if (semanticSchemaResp == null) { return new QueryStatement(); } - Map modelSchemaRespMap = modelSchemaResps.stream() - .collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp)); //1.convert name to bizName - convertNameToBizName(querySqlReq, modelSchemaResps); + convertNameToBizName(querySQLReq, semanticSchemaResp); //2.functionName corrector - functionNameCorrector(querySqlReq); + functionNameCorrector(querySQLReq, semanticSchemaResp); //3.correct tableName - correctTableName(querySqlReq); + correctTableName(querySQLReq); - String tableName = SqlParserSelectHelper.getTableName(querySqlReq.getSql()); + String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql()); if (StringUtils.isEmpty(tableName)) { return new QueryStatement(); } //4.build MetricTables - List allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()); - List metrics = getMetrics(modelSchemaResps, allFields); + List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); + List metrics = getMetrics(semanticSchemaResp, allFields); QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); metricTable.setMetrics(metrics); - Set dimensions = getDimensions(modelSchemaResps, allFields); + Set dimensions = getDimensions(semanticSchemaResp, allFields); metricTable.setDimensions(new ArrayList<>(dimensions)); @@ -96,36 +91,33 @@ public class QueryReqConverter { // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { metricTable.setMetrics(new ArrayList<>()); - for (Long modelId : querySqlReq.getModelIds()) { - ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName())); - } + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + semanticSchemaResp.getModelResps().get(0).getBizName())); } else { queryStructReq.setAggregators( metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( Collectors.toList())); } - AggOption aggOption = getAggOption(querySqlReq); + AggOption aggOption = getAggOption(querySQLReq); metricTable.setAggOption(aggOption); List tables = new ArrayList<>(); tables.add(metricTable); //4.build ParseSqlReq ParseSqlReq result = new ParseSqlReq(); - BeanUtils.copyProperties(querySqlReq, result); + BeanUtils.copyProperties(querySQLReq, result); - result.setRootPath(querySqlReq.getModelIdStr()); result.setTables(tables); - DatabaseResp database = catalog.getDatabaseByModelId(querySqlReq.getModelIds().get(0)); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); } //5. do deriveMetric - generateDerivedMetric(querySqlReq.getModelIds(), modelSchemaResps, aggOption, result); + generateDerivedMetric(semanticSchemaResp, aggOption, result); //6.physicalSql by ParseSqlReq - queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); - queryStructReq.setModelIds(new HashSet<>(querySqlReq.getModelIds())); + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); + queryStructReq.setViewId(querySQLReq.getViewId()); queryStructReq.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); QueryStatement queryStatement = new QueryStatement(); @@ -133,7 +125,7 @@ public class QueryReqConverter { queryStatement.setParseSqlReq(result); queryStatement.setIsS2SQL(true); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setModelIds(querySqlReq.getModelIds()); + queryStatement.setViewId(querySQLReq.getViewId()); queryStatement.setEnableLimitWrapper(limitWrapper); return queryStatement; @@ -151,8 +143,8 @@ public class QueryReqConverter { return AggOption.DEFAULT; } - private void convertNameToBizName(QuerySqlReq databaseReq, List modelSchemaResps) { - Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps); + private void convertNameToBizName(QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) { + Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); @@ -160,9 +152,8 @@ public class QueryReqConverter { databaseReq.setSql(replaceFields); } - private Set getDimensions(List modelSchemaResps, List allFields) { - Map dimensionLowerToNameMap = modelSchemaResps.stream() - .flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream()) + private Set getDimensions(SemanticSchemaResp semanticSchemaResp, List allFields) { + Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName, (k1, k2) -> k1)); Map internalLowerToNameMap = QueryStructUtils.internalCols.stream() @@ -173,16 +164,15 @@ public class QueryReqConverter { .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet()); } - private List getMetrics(List modelSchemaResps, List allFields) { - Map metricLowerToNameMap = modelSchemaResps.stream() - .flatMap(modelSchemaResp -> modelSchemaResp.getMetrics().stream()) + private List getMetrics(SemanticSchemaResp semanticSchemaResp, List allFields) { + Map metricLowerToNameMap = semanticSchemaResp.getMetrics().stream() .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName)); return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); } - private void functionNameCorrector(QuerySqlReq databaseReq) { - DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0)); + private void functionNameCorrector(QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) { + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); if (Objects.isNull(database) || Objects.isNull(database.getType())) { return; } @@ -194,15 +184,13 @@ public class QueryReqConverter { } } - protected Map getFieldNameToBizNameMap(List modelSchemaResps) { + protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { // support fieldName and field alias to bizName - Map dimensionResults = modelSchemaResps.stream().flatMap(modelSchemaResp - -> modelSchemaResp.getDimensions().stream()) + Map dimensionResults = semanticSchemaResp.getDimensions().stream() .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - Map metricResults = modelSchemaResps.stream().flatMap(modelSchemaResp - -> modelSchemaResp.getMetrics().stream()) + Map metricResults = semanticSchemaResp.getMetrics().stream() .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); @@ -224,12 +212,11 @@ public class QueryReqConverter { return elements.stream(); } - public void correctTableName(QuerySqlReq databaseReq) { - String sql = databaseReq.getSql(); - for (Long modelId : databaseReq.getModelIds()) { - sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId); - } - databaseReq.setSql(sql); + public void correctTableName(QuerySqlReq querySqlReq) { + String sql = querySqlReq.getSql(); + sql = SqlParserReplaceHelper.replaceTable(sql, + Constants.TABLE_PREFIX + querySqlReq.getViewId()); + querySqlReq.setSql(sql); } private QueryType getQueryType(AggOption aggOption) { @@ -241,13 +228,13 @@ public class QueryReqConverter { return queryType; } - private void generateDerivedMetric(List modelIds, List modelSchemaResps, AggOption aggOption, + private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, ParseSqlReq parseSqlReq) { String sql = parseSqlReq.getSql(); for (MetricTable metricTable : parseSqlReq.getTables()) { List measures = new ArrayList<>(); Map replaces = new HashMap<>(); - generateDerivedMetric(modelIds, modelSchemaResps, aggOption, metricTable.getMetrics(), + generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(), metricTable.getDimensions(), measures, replaces); if (!CollectionUtils.isEmpty(replaces)) { @@ -263,13 +250,11 @@ public class QueryReqConverter { parseSqlReq.setSql(sql); } - private void generateDerivedMetric(List modelIds, List modelSchemaResps, AggOption aggOption, + private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List metrics, List dimensions, List measures, Map replaces) { - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setModelIds(modelIds); - List metricResps = catalog.getMetrics(metaFilter); - List dimensionResps = catalog.getDimensions(metaFilter); + List metricResps = semanticSchemaResp.getMetrics(); + List dimensionResps = semanticSchemaResp.getDimensions(); // check metrics has derived if (!metricResps.stream() .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(), @@ -279,14 +264,13 @@ public class QueryReqConverter { log.info("begin to generateDerivedMetric {} [{}]", aggOption, metrics); Set allFields = new HashSet<>(); Map allMeasures = new HashMap<>(); - modelSchemaResps.stream().forEach(modelSchemaResp -> { - allFields.addAll(modelSchemaResp.getFieldList()); - if (Objects.nonNull(modelSchemaResp.getModelDetail().getMeasures())) { - modelSchemaResp.getModelDetail().getMeasures().stream() + semanticSchemaResp.getModelResps().forEach(modelResp -> { + allFields.addAll(modelResp.getFieldList()); + if (Objects.nonNull(modelResp.getModelDetail().getMeasures())) { + modelResp.getModelDetail().getMeasures().stream() .forEach(mm -> allMeasures.put(mm.getBizName(), mm)); } }); - Set deriveDimension = new HashSet<>(); Set deriveMetric = new HashSet<>(); Set visitedMetric = new HashSet<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 32d439da4..4cd3ea92a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -1,10 +1,5 @@ package com.tencent.supersonic.headless.server.utils; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; @@ -22,9 +17,16 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -38,11 +40,11 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; @Slf4j @@ -65,26 +67,27 @@ public class QueryStructUtils { public QueryStructUtils( DateModeUtils dateModeUtils, - SqlFilterUtils sqlFilterUtils, Catalog catalog) { + SqlFilterUtils sqlFilterUtils, @Lazy Catalog catalog) { this.dateModeUtils = dateModeUtils; this.sqlFilterUtils = sqlFilterUtils; this.catalog = catalog; } - private List getDimensionIds(QueryStructReq queryStructCmd) { + private List getDimensionIds(QueryStructReq queryStructReq) { List dimensionIds = new ArrayList<>(); - MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds()); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setViewId(queryStructReq.getViewId()); List dimensions = catalog.getDimensions(metaFilter); Map> pair = dimensions.stream() .collect(Collectors.groupingBy(DimensionResp::getBizName)); - for (String group : queryStructCmd.getGroups()) { + for (String group : queryStructReq.getGroups()) { if (pair.containsKey(group)) { dimensionIds.add(pair.get(group).get(0).getId()); } } - List filtersCols = sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()); + List filtersCols = sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()); for (String col : filtersCols) { if (pair.containsKey(col)) { dimensionIds.add(pair.get(col).get(0).getId()); @@ -95,7 +98,8 @@ public class QueryStructUtils { private List getMetricIds(QueryStructReq queryStructCmd) { List metricIds = new ArrayList<>(); - MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds()); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setViewId(queryStructCmd.getViewId()); List metrics = catalog.getMetrics(metaFilter); Map> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName)); for (Aggregator agg : queryStructCmd.getAggregators()) { @@ -133,12 +137,12 @@ public class QueryStructUtils { } public Set getResNameEnExceptInternalCol(QuerySqlReq querySqlReq, - List modelSchemaRespList) { + SemanticSchemaResp semanticSchemaResp) { Set resNameSet = getResName(querySqlReq); Set resNameEnSet = new HashSet<>(); - if (!CollectionUtils.isEmpty(modelSchemaRespList)) { - List metrics = modelSchemaRespList.get(0).getMetrics(); - List dimensions = modelSchemaRespList.get(0).getDimensions(); + if (semanticSchemaResp != null) { + List metrics = semanticSchemaResp.getMetrics(); + List dimensions = semanticSchemaResp.getDimensions(); metrics.stream().forEach(o -> { if (resNameSet.contains(o.getName())) { resNameEnSet.add(o.getBizName()); @@ -174,8 +178,8 @@ public class QueryStructUtils { List dimensionIds = getDimensionIds(queryStructCmd); List metricIds = getMetricIds(queryStructCmd); ItemDateResp dateDate = catalog.getItemDate( - new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()), - new ItemDateFilter(metricIds, TypeEnums.METRIC.getName())); + new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.name()), + new ItemDateFilter(metricIds, TypeEnums.METRIC.name())); return dateDate; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 460bf302f..158786c21 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -1,20 +1,23 @@ package com.tencent.supersonic.headless.server.utils; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; - import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.Catalog; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -24,11 +27,9 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; + +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; @Slf4j @@ -43,11 +44,6 @@ public class QueryUtils { @Value("${query.optimizer.enable:true}") private Boolean optimizeEnable; - private final Catalog catalog; - - public QueryUtils(Catalog catalog) { - this.catalog = catalog; - } @PostConstruct public void fillPattern() { @@ -58,10 +54,9 @@ public class QueryUtils { } } - public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, List modelIds) { - MetaFilter metaFilter = new MetaFilter(modelIds); - List metricDescList = catalog.getMetrics(metaFilter); - List dimensionDescList = catalog.getDimensions(metaFilter); + public void fillItemNameInfo(SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) { + List metricDescList = semanticSchemaResp.getMetrics(); + List dimSchemaResps = semanticSchemaResp.getDimensions(); Map metricRespMap = metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); Map namePair = new HashMap<>(); @@ -71,11 +66,11 @@ public class QueryUtils { namePair.put(metricDesc.getBizName(), metricDesc.getName()); nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name()); }); - dimensionDescList.forEach(dimensionDesc -> { + dimSchemaResps.forEach(dimensionDesc -> { namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName()); nameTypePair.put(dimensionDesc.getBizName(), dimensionDesc.getSemanticType()); }); - List columns = queryResultWithColumns.getColumns(); + List columns = semanticQueryResp.getColumns(); columns.forEach(column -> { String nameEn = getName(column.getNameEn().toLowerCase()); if (nameEn.contains(JOIN_UNDERLINE)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index bac5e6602..97a12d37a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -7,18 +7,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.headless.api.pojo.QueryStat; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.QueryOptMode; import com.tencent.supersonic.headless.api.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack; -import com.tencent.supersonic.headless.api.pojo.QueryStat; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.service.ModelService; import lombok.extern.slf4j.Slf4j; @@ -101,22 +100,15 @@ public class StatUtils { public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); + List aggFields = SqlParserSelectHelper.getAggregateFields(querySqlReq.getSql()); List allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()); - queryStatInfo.setModelId(querySqlReq.getModelIds().get(0)); - ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySqlReq.getModelIds().get(0)); - - List dimensions = new ArrayList<>(); - List metrics = new ArrayList<>(); - if (Objects.nonNull(modelSchemaResp)) { - dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions()); - metrics = getFieldNames(allFields, modelSchemaResp.getMetrics()); - } + List dimensions = allFields.stream().filter(aggFields::contains).collect(Collectors.toList()); String userName = getUserName(facadeUser); try { queryStatInfo.setTraceId("") - .setModelId(querySqlReq.getModelIds().get(0)) .setUser(userName) + .setViewId(querySqlReq.getViewId()) .setQueryType(QueryType.SQL.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQuerySqlCmd(querySqlReq.toString()) @@ -124,43 +116,49 @@ public class StatUtils { .setStartTime(System.currentTimeMillis()) .setUseResultCache(true) .setUseSqlCache(true) - .setMetrics(objectMapper.writeValueAsString(metrics)) + .setMetrics(objectMapper.writeValueAsString(aggFields)) .setDimensions(objectMapper.writeValueAsString(dimensions)); + if (!CollectionUtils.isEmpty(querySqlReq.getModelIds())) { + queryStatInfo.setModelId(querySqlReq.getModelIds().get(0)); + } } catch (JsonProcessingException e) { log.error("initStatInfo:{}", e); } StatUtils.set(queryStatInfo); } - public void initStructStatInfo(QueryStructReq queryStructCmd, User facadeUser) { + public void initStructStatInfo(QueryStructReq queryStructReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); String traceId = ""; - List dimensions = queryStructCmd.getGroups(); + List dimensions = queryStructReq.getGroups(); List metrics = new ArrayList<>(); - queryStructCmd.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); + queryStructReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); String user = getUserName(facadeUser); try { queryStatInfo.setTraceId(traceId) - .setModelId(1L) + .setViewId(queryStructReq.getViewId()) .setUser(user) .setQueryType(QueryType.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) - .setQueryStructCmd(queryStructCmd.toString()) - .setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString())) + .setQueryStructCmd(queryStructReq.toString()) + .setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructReq.toString())) .setStartTime(System.currentTimeMillis()) - .setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery()) - .setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups())) - .setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators())) - .setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders())) + .setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery()) + .setGroupByCols(objectMapper.writeValueAsString(queryStructReq.getGroups())) + .setAggCols(objectMapper.writeValueAsString(queryStructReq.getAggregators())) + .setOrderByCols(objectMapper.writeValueAsString(queryStructReq.getOrders())) .setFilterCols(objectMapper.writeValueAsString( - sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()))) + sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()))) .setUseResultCache(true) .setUseSqlCache(true) .setMetrics(objectMapper.writeValueAsString(metrics)) .setDimensions(objectMapper.writeValueAsString(dimensions)) .setQueryOptMode(QueryOptMode.NONE.name()); + if (!CollectionUtils.isEmpty(queryStructReq.getModelIds())) { + queryStatInfo.setModelId(queryStructReq.getModelIds().get(0)); + } } catch (JsonProcessingException e) { e.printStackTrace(); } diff --git a/headless/server/src/main/resources/mapper/StatMapper.xml b/headless/server/src/main/resources/mapper/StatMapper.xml index 57bc055c0..e1b4d1a90 100644 --- a/headless/server/src/main/resources/mapper/StatMapper.xml +++ b/headless/server/src/main/resources/mapper/StatMapper.xml @@ -9,6 +9,7 @@ + @@ -42,13 +43,13 @@ insert into s2_query_stat_info ( - trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine, + trace_id, model_id, view_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine, elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols, order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode ) values ( - #{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine}, + #{traceId}, #{modelId}, #{viewId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine}, #{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols}, #{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode} ) diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 81479624e..e53600403 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -29,12 +29,8 @@ class HeadlessParserServiceTest { private static Map headlessSchemaMap = new HashMap<>(); - public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricCommand, boolean isAgg) { + public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricQueryReq, boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); - if (metricCommand.getRootPath().isEmpty()) { - sqlParser.setErrMsg("rootPath empty"); - return sqlParser; - } try { if (semanticSchema == null) { sqlParser.setErrMsg("headlessSchema not found"); @@ -42,14 +38,14 @@ class HeadlessParserServiceTest { } AggPlanner aggBuilder = new AggPlanner(semanticSchema); QueryStatement queryStatement = new QueryStatement(); - queryStatement.setMetricReq(metricCommand); + queryStatement.setMetricReq(metricQueryReq); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); sqlParser.setSql(aggBuilder.getSql(engineType)); sqlParser.setSourceId(aggBuilder.getSourceId()); } catch (Exception e) { sqlParser.setErrMsg(e.getMessage()); - log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e); + log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e); } return sqlParser; } @@ -123,8 +119,7 @@ class HeadlessParserServiceTest { identify.setType("primary"); identifies.add(identify); datasource.setIdentifiers(identifies); - - SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build(); + SemanticSchema semanticSchema = SemanticSchema.newBuilder("1").build(); SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource)); @@ -168,7 +163,6 @@ class HeadlessParserServiceTest { //HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); MetricQueryReq metricCommand = new MetricQueryReq(); - metricCommand.setRootPath("s2"); metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); @@ -181,7 +175,6 @@ class HeadlessParserServiceTest { addDepartment(semanticSchema); MetricQueryReq metricCommand2 = new MetricQueryReq(); - metricCommand2.setRootPath("s2"); metricCommand2.setDimensions(new ArrayList<>( Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java index f126b9a58..969b140ec 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java @@ -1,13 +1,7 @@ package com.tencent.supersonic.headless.server.service; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -import com.alibaba.excel.util.FileUtils; import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq; @@ -15,28 +9,18 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl; -import java.io.File; -import java.util.Date; +import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; class DownloadServiceImplTest { @Test void testBatchDownload() throws Exception { - ModelService modelService = Mockito.mock(ModelService.class); - QueryService queryService = Mockito.mock(QueryService.class); - when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp())); - when(queryService.queryByReq(any(), any())).thenReturn(mockQueryResult()); - DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService); - String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); - File file = FileUtils.createTmpFile(fileName); - downloadService.batchDownload(buildBatchDownloadReq(), User.getFakeUser(), file); + } private ModelSchemaResp mockModelSchemaResp() { diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index 663a3cfd8..828f98724 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -20,6 +20,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl; +import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl; import com.tencent.supersonic.headless.server.utils.MetricConverter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -65,8 +66,9 @@ public class MetricServiceImplTest { ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class); CollectService collectService = Mockito.mock(CollectService.class); ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); + ViewService viewService = Mockito.mock(ViewServiceImpl.class); return new MetricServiceImpl(metricRepository, modelService, domainService, - chatGptHelper, collectService, eventPublisher); + chatGptHelper, collectService, viewService, eventPublisher); } private MetricReq buildMetricReq() { diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java index 2e91dff47..16baa480d 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java @@ -76,8 +76,10 @@ class ModelServiceImplTest { UserService userService = Mockito.mock(UserService.class); ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class); DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); + ViewService viewService = Mockito.mock(ViewService.class); return new ModelServiceImpl(modelRepository, databaseService, - dimensionService, metricService, modelRelaService, domainService, userService, dateInfoRepository); + dimensionService, metricService, modelRelaService, domainService, + userService, viewService, dateInfoRepository); } private ModelReq mockModelReq() { diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index b41c7946e..c9ff38195 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -21,7 +21,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter -com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\ +com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index aaf803c1b..7ca3f6d3f 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -3,15 +3,8 @@ package com.tencent.supersonic; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; -import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; -import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.core.agent.Agent; import com.tencent.supersonic.chat.core.agent.AgentConfig; @@ -39,9 +32,7 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; @Component @@ -82,14 +73,6 @@ public class ChatDemoLoader implements CommandLineRunner { public void doRun() { try { addSysParameter(); - addDemoChatConfig_1(); - addDemoChatConfig_2(); - addDemoChatConfig_3(); - addDemoChatConfig_4(); - addDemoChatConfig_5(); - addDemoChatConfig_6(); - addDemoChatConfig_7(); - addDemoChatConfig_8(); addPlugin_1(); addAgent1(); addAgent2(); @@ -147,275 +130,10 @@ public class ChatDemoLoader implements CommandLineRunner { sysParameterService.save(sysParameter); } - public void addDemoChatConfig_1() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(1L); - - ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); - ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = Collections.singletonList(1L); - List metricIds0 = Lists.newArrayList(); - chatDefaultConfigDetail.setDimensionIds(dimensionIds0); - chatDefaultConfigDetail.setMetricIds(metricIds0); - chatDefaultConfigDetail.setUnit(7); - chatDefaultConfigDetail.setPeriod("DAY"); - chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); - ItemVisibility visibility0 = new ItemVisibility(); - chatDetailConfig.setVisibility(visibility0); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - - ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); - ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = Arrays.asList(1L); - List metricIds1 = Lists.newArrayList(); - chatDefaultConfigAgg.setDimensionIds(dimensionIds1); - chatDefaultConfigAgg.setMetricIds(metricIds1); - chatDefaultConfigAgg.setUnit(7); - chatDefaultConfigAgg.setPeriod("DAY"); - chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); - chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); - ItemVisibility visibility1 = new ItemVisibility(); - chatAggConfig.setVisibility(visibility1); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setItemId(3L); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfos.add(knowledgeInfoReq); - chatAggConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatAggConfig(chatAggConfig); - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_2() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(2L); - - ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); - ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = new ArrayList<>(); - List metricIds0 = Arrays.asList(1L, 2L); - chatDefaultConfigDetail.setDimensionIds(dimensionIds0); - chatDefaultConfigDetail.setMetricIds(metricIds0); - chatDefaultConfigDetail.setUnit(7); - chatDefaultConfigDetail.setPeriod("DAY"); - chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); - ItemVisibility visibility0 = new ItemVisibility(); - chatDetailConfig.setVisibility(visibility0); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - - ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); - ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = new ArrayList<>(); - List metricIds1 = Arrays.asList(1L, 2L); - chatDefaultConfigAgg.setDimensionIds(dimensionIds1); - chatDefaultConfigAgg.setMetricIds(metricIds1); - chatDefaultConfigAgg.setUnit(7); - chatDefaultConfigAgg.setPeriod("DAY"); - chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); - chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); - ItemVisibility visibility1 = new ItemVisibility(); - chatAggConfig.setVisibility(visibility1); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setItemId(3L); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfos.add(knowledgeInfoReq); - chatAggConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatAggConfig(chatAggConfig); - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_3() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(3L); - - ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); - ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = Collections.singletonList(3L); - List metricIds0 = Collections.singletonList(4L); - chatDefaultConfigDetail.setDimensionIds(dimensionIds0); - chatDefaultConfigDetail.setMetricIds(metricIds0); - chatDefaultConfigDetail.setUnit(7); - chatDefaultConfigDetail.setPeriod("DAY"); - chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); - ItemVisibility visibility0 = new ItemVisibility(); - chatDetailConfig.setVisibility(visibility0); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - - ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); - ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = Collections.singletonList(3L); - List metricIds1 = Collections.singletonList(4L); - chatDefaultConfigAgg.setDimensionIds(dimensionIds1); - chatDefaultConfigAgg.setMetricIds(metricIds1); - chatDefaultConfigAgg.setUnit(7); - chatDefaultConfigAgg.setPeriod("DAY"); - chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); - chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); - ItemVisibility visibility1 = new ItemVisibility(); - chatAggConfig.setVisibility(visibility1); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setItemId(5L); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfos.add(knowledgeInfoReq); - chatAggConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatAggConfig(chatAggConfig); - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_4() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(4L); - - ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); - ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = Arrays.asList(4L, 5L, 6L, 7L); - List metricIds0 = Arrays.asList(5L); - chatDefaultConfigDetail.setDimensionIds(dimensionIds0); - chatDefaultConfigDetail.setMetricIds(metricIds0); - chatDefaultConfigDetail.setUnit(7); - chatDefaultConfigDetail.setPeriod("DAY"); - chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); - ItemVisibility visibility0 = new ItemVisibility(); - chatDetailConfig.setVisibility(visibility0); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - - ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); - ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = Arrays.asList(4L, 4L, 6L, 7L); - List metricIds1 = Arrays.asList(5L); - chatDefaultConfigAgg.setDimensionIds(dimensionIds1); - chatDefaultConfigAgg.setMetricIds(metricIds1); - chatDefaultConfigAgg.setUnit(7); - chatDefaultConfigAgg.setPeriod("DAY"); - chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); - chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); - ItemVisibility visibility1 = new ItemVisibility(); - chatAggConfig.setVisibility(visibility1); - chatConfigBaseReq.setChatAggConfig(chatAggConfig); - - List recommendedQuestions = new ArrayList<>(); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_5() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(5L); - - ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfoReq.setItemId(10L); - knowledgeInfoReq.setBizName("most_popular_in"); - knowledgeInfos.add(knowledgeInfoReq); - - KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); - knowledgeInfoReq2.setSearchEnable(true); - knowledgeInfoReq2.setItemId(11L); - knowledgeInfoReq2.setBizName("g_name"); - knowledgeInfos.add(knowledgeInfoReq2); - - chatDetailConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - List recommendedQuestions = new ArrayList<>(); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - configService.addConfig(chatConfigBaseReq, user); - } - - private ChatDetailConfigReq getChatDetailConfigReq() { - ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); - ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - chatDefaultConfigDetail.setUnit(-1); - chatDefaultConfigDetail.setPeriod("DAY"); - chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); - ItemVisibility visibility0 = new ItemVisibility(); - chatDetailConfig.setVisibility(visibility0); - return chatDetailConfig; - } - - public void addDemoChatConfig_6() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(6L); - - ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfoReq.setItemId(12L); - knowledgeInfoReq.setBizName("country"); - knowledgeInfos.add(knowledgeInfoReq); - - KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); - knowledgeInfoReq2.setSearchEnable(true); - knowledgeInfoReq2.setItemId(13L); - knowledgeInfoReq2.setBizName("gender"); - knowledgeInfos.add(knowledgeInfoReq2); - - chatDetailConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - List recommendedQuestions = new ArrayList<>(); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_7() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(7L); - - ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfoReq.setItemId(16L); - knowledgeInfoReq.setBizName("formats"); - knowledgeInfos.add(knowledgeInfoReq); - - chatDetailConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - List recommendedQuestions = new ArrayList<>(); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - configService.addConfig(chatConfigBaseReq, user); - } - - public void addDemoChatConfig_8() { - ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(8L); - - ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); - List knowledgeInfos = new ArrayList<>(); - KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setSearchEnable(true); - knowledgeInfoReq.setItemId(18L); - knowledgeInfoReq.setBizName("country"); - knowledgeInfos.add(knowledgeInfoReq); - - KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); - knowledgeInfoReq2.setSearchEnable(true); - knowledgeInfoReq2.setItemId(19L); - knowledgeInfoReq2.setBizName("languages"); - knowledgeInfos.add(knowledgeInfoReq2); - - KnowledgeInfoReq knowledgeInfoReq3 = new KnowledgeInfoReq(); - knowledgeInfoReq3.setSearchEnable(true); - knowledgeInfoReq3.setItemId(21L); - knowledgeInfoReq3.setBizName("song_name"); - knowledgeInfos.add(knowledgeInfoReq3); - - chatDetailConfig.setKnowledgeInfos(knowledgeInfos); - chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); - List recommendedQuestions = new ArrayList<>(); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - configService.addConfig(chatConfigBaseReq, user); - } - private void addPlugin_1() { Plugin plugin1 = new Plugin(); plugin1.setType("WEB_PAGE"); - plugin1.setModelList(Arrays.asList(1L)); + plugin1.setViewList(Arrays.asList(1L)); plugin1.setPattern("用于分析超音数的流量概况，包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示，无实际看板"); plugin1.setName("超音数流量分析看板"); PluginParseConfig pluginParseConfig = new PluginParseConfig(); @@ -450,15 +168,14 @@ public class ChatDemoLoader implements CommandLineRunner { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); ruleQueryTool.setId("0"); - ruleQueryTool.setModelIds(Lists.newArrayList(-1L)); + ruleQueryTool.setViewIds(Lists.newArrayList(-1L)); ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.METRIC.name())); agentConfig.getTools().add(ruleQueryTool); - if (demoEnabledNl2SqlLlm) { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setModelIds(Lists.newArrayList(-1L)); + llmParserTool.setViewIds(Lists.newArrayList(-1L)); agentConfig.getTools().add(llmParserTool); } agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); @@ -477,7 +194,7 @@ public class ChatDemoLoader implements CommandLineRunner { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setId("0"); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); - ruleQueryTool.setModelIds(Lists.newArrayList(-1L)); + ruleQueryTool.setViewIds(Lists.newArrayList(-1L)); ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.TAG.name())); agentConfig.getTools().add(ruleQueryTool); @@ -485,7 +202,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setModelIds(Lists.newArrayList(-1L)); + llmParserTool.setViewIds(Lists.newArrayList(-1L)); agentConfig.getTools().add(llmParserTool); } agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); @@ -507,7 +224,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setModelIds(Lists.newArrayList(5L, 6L, 7L, 8L)); + llmParserTool.setViewIds(Lists.newArrayList(5L, 6L, 7L, 8L)); agentConfig.getTools().add(llmParserTool); } @@ -529,7 +246,7 @@ public class ChatDemoLoader implements CommandLineRunner { LLMParserTool llmParserTool = new LLMParserTool(); llmParserTool.setId("1"); llmParserTool.setType(AgentToolType.NL2SQL_LLM); - llmParserTool.setModelIds(Lists.newArrayList(9L, 10L, 11L, 12L)); + llmParserTool.setViewIds(Lists.newArrayList(9L, 10L, 11L, 12L)); agentConfig.getTools().add(llmParserTool); } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index e19f70515..874efb78a 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -12,6 +12,12 @@ import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.enums.DataType; import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; @@ -31,17 +37,21 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.RelateDimension; +import com.tencent.supersonic.headless.api.pojo.ViewDetail; +import com.tencent.supersonic.headless.api.pojo.ViewModelConfig; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; +import com.tencent.supersonic.headless.api.pojo.request.ViewReq; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.ViewService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -73,6 +83,8 @@ public class ModelDemoDataLoader { @Autowired private AuthService authService; @Autowired + private ViewService viewService; + @Autowired private DataSourceProperties dataSourceProperties; public void doRun() { @@ -91,6 +103,8 @@ public class ModelDemoDataLoader { updateDimension(); updateMetric(); updateMetric_pv(); + addView_1(); + addView_2(); addAuthGroup_1(); addAuthGroup_2(); } catch (Exception e) { @@ -429,6 +443,65 @@ public class ModelDemoDataLoader { metricService.createMetric(metricReq, user); } + public void addView_1() { + ViewReq viewReq = new ViewReq(); + viewReq.setName("超音数"); + viewReq.setBizName("s2"); + viewReq.setDomainId(1L); + viewReq.setDescription("包含超音数访问统计相关的指标和维度等"); + viewReq.setAdmins(Lists.newArrayList("admin")); + List viewModelConfigs = Lists.newArrayList( + new ViewModelConfig(1L, true), + new ViewModelConfig(2L, true), + new ViewModelConfig(3L, true)); + + ViewDetail viewDetail = new ViewDetail(); + viewDetail.setViewModelConfigs(viewModelConfigs); + viewReq.setViewDetail(viewDetail); + viewReq.setTypeEnum(TypeEnums.VIEW); + QueryConfig queryConfig = new QueryConfig(); + MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); + TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); + timeDefaultConfig.setTimeMode(TimeMode.RECENT); + timeDefaultConfig.setUnit(7); + metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig); + queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig); + viewReq.setQueryConfig(queryConfig); + viewService.save(viewReq, User.getFakeUser()); + } + + public void addView_2() { + ViewReq viewReq = new ViewReq(); + viewReq.setName("艺人库"); + viewReq.setBizName("singer"); + viewReq.setDomainId(2L); + viewReq.setDescription("包含艺人相关标签和指标信息"); + viewReq.setAdmins(Lists.newArrayList("admin")); + List viewModelConfigs = + Lists.newArrayList(new ViewModelConfig(4L, true)); + ViewDetail viewDetail = new ViewDetail(); + viewDetail.setViewModelConfigs(viewModelConfigs); + viewReq.setViewDetail(viewDetail); + viewReq.setTypeEnum(TypeEnums.VIEW); + QueryConfig queryConfig = new QueryConfig(); + TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig(); + TimeDefaultConfig tagTimeDefaultConfig = new TimeDefaultConfig(); + tagTimeDefaultConfig.setTimeMode(TimeMode.LAST); + tagTimeDefaultConfig.setUnit(7); + tagTypeDefaultConfig.setTimeDefaultConfig(tagTimeDefaultConfig); + tagTypeDefaultConfig.setDimensionIds(Lists.newArrayList(4L, 5L, 6L, 7L)); + tagTypeDefaultConfig.setMetricIds(Lists.newArrayList(5L)); + MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig(); + TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); + timeDefaultConfig.setTimeMode(TimeMode.RECENT); + timeDefaultConfig.setUnit(7); + metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig); + queryConfig.setTagTypeDefaultConfig(tagTypeDefaultConfig); + queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig); + viewReq.setQueryConfig(queryConfig); + viewService.save(viewReq, User.getFakeUser()); + } + public void addAuthGroup_1() { AuthGroup authGroupReq = new AuthGroup(); authGroupReq.setModelId(3L); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 7e11c81ce..80b5b0431 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -2,8 +2,7 @@ com.tencent.supersonic.chat.core.mapper.SchemaMapper=\ com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \ com.tencent.supersonic.chat.core.mapper.KeywordMapper, \ com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.core.mapper.EntityMapper, \ - com.tencent.supersonic.chat.core.mapper.ModelClusterMapper + com.tencent.supersonic.chat.core.mapper.EntityMapper com.tencent.supersonic.chat.core.parser.SemanticParser=\ com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \ @@ -17,8 +16,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.corrector.SelectCorrector, \ com.tencent.supersonic.chat.core.corrector.WhereCorrector, \ com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \ - com.tencent.supersonic.chat.core.corrector.HavingCorrector, \ - com.tencent.supersonic.chat.core.corrector.FromCorrector + com.tencent.supersonic.chat.core.corrector.HavingCorrector com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.server.processor.parse.MetricCheckProcessor, \ @@ -33,8 +31,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter -com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\ - com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver +com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ + com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index 2288b7f0d..fffd78699 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -168,4 +168,25 @@ CREATE TABLE `s2_app` --20240115 alter table s2_metric add column `define_type` varchar(50) DEFAULT NULL; -- MEASURE, FIELD, METRIC -update s2_metric set define_type = 'MEASURE'; \ No newline at end of file +update s2_metric set define_type = 'MEASURE'; + +--20240129 +CREATE TABLE s2_view( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + domain_id BIGINT, + `name` VARCHAR(255), + biz_name VARCHAR(255), + `description` VARCHAR(255), + `status` INT, + alias VARCHAR(255), + view_detail text, + created_at datetime, + created_by VARCHAR(255), + updated_at datetime, + updated_by VARCHAR(255) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +alter table s2_plugin change column model `view` varchar(100); +alter table s2_view_info rename to s2_canvas; + +alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt index b01cf97ae..9871036d4 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _4_7 100 -陈奕迅 _4_7 100 -林俊杰 _4_7 100 -张碧晨 _4_7 100 -程响 _4_7 100 -Taylor#Swift _4_7 100 -内地 _4_4 100 -欧美 _4_4 100 -港台 _4_4 100 -流行 _4_6 100 -国风 _4_6 100 \ No newline at end of file +周杰伦 _2_7 100 +陈奕迅 _2_7 100 +林俊杰 _2_7 100 +张碧晨 _2_7 100 +程响 _2_7 100 +Taylor#Swift _2_7 100 +内地 _2_4 100 +欧美 _2_4 100 +港台 _2_4 100 +流行 _2_6 100 +国风 _2_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt index e442a219c..d0d411751 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _3_3 52 -p2 _3_3 47 -p3 _3_3 31 -p4 _3_3 36 -p5 _3_3 50 -p6 _3_3 38 \ No newline at end of file +p1 _2_3 52 +p2 _2_3 47 +p3 _2_3 31 +p4 _2_3 36 +p5 _2_3 50 +p6 _2_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt index 3bc23d613..6699a0fc8 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _4_7 9000 -周深 _4_7 8000 -周传雄 _4_7 7000 -周华建 _4_7 6000 -陈奕迅 _4_7 8000 -林俊杰 _4_7 7000 -张碧晨 _4_7 7000 -程响 _4_7 7000 -Taylor#Swift _4_7 7000 \ No newline at end of file +周杰伦 _2_7 9000 +周深 _2_7 8000 +周传雄 _2_7 7000 +周华建 _2_7 6000 +陈奕迅 _2_7 8000 +林俊杰 _2_7 7000 +张碧晨 _2_7 7000 +程响 _2_7 7000 +Taylor#Swift _2_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/data-h2.sql b/launchers/standalone/src/main/resources/db/data-h2.sql index 72e046101..fb8c36f23 100644 --- a/launchers/standalone/src/main/resources/db/data-h2.sql +++ b/launchers/standalone/src/main/resources/db/data-h2.sql @@ -12,7 +12,7 @@ values (2 , 2, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), D MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (3 , 3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -MERGE INTO s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) +MERGE INTO s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -- sample data diff --git a/launchers/standalone/src/main/resources/db/data-mysql.sql b/launchers/standalone/src/main/resources/db/data-mysql.sql index 44137c9be..1844d9f57 100644 --- a/launchers/standalone/src/main/resources/db/data-mysql.sql +++ b/launchers/standalone/src/main/resources/db/data-mysql.sql @@ -12,7 +12,7 @@ VALUES (2, 'dimension', 'yyyy-MM-dd', DATE_SUB(CURRENT_DATE(), INTERVAL 28 DAY), INSERT INTO s2_available_date_info (`item_id`, `type`, `date_format`, `start_date`, `end_date`, `unavailable_date`, `created_at`, `created_by`, `updated_at`, `updated_by`) VALUES (3, 'dimension', 'yyyy-MM-dd', DATE_SUB(CURRENT_DATE(), INTERVAL 28 DAY), DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -insert into s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) +insert into s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -- sample data diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 9b653cbcf..5b7ac3b30 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -230,8 +230,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela PRIMARY KEY (`id`) ); -create table IF NOT EXISTS s2_view_info -( +create table IF NOT EXISTS `s2_canvas` ( id INT auto_increment, domain_id INT null, type varchar(20) null comment 'model、dimension、metric', @@ -242,13 +241,14 @@ create table IF NOT EXISTS s2_view_info updated_by varchar(100) not null, PRIMARY KEY (`id`) ); -COMMENT ON TABLE s2_view_info IS 'view information table'; +COMMENT ON TABLE s2_canvas IS 'canvas table'; CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( `id` INT NOT NULL AUTO_INCREMENT, `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier `model_id` INT DEFAULT NULL, + `view_id` INT DEFAULT NULL, `user` varchar(200) DEFAULT NULL, `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene @@ -326,7 +326,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` ( `id` INT AUTO_INCREMENT, `type` varchar(50) NULL, - `model` varchar(100) NULL, + `view` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, `parse_mode_config` LONGVARCHAR NULL, @@ -544,3 +544,22 @@ CREATE TABLE IF NOT EXISTS `s2_app` ( updated_at TIMESTAMP, updated_by VARCHAR(255) ); + +CREATE TABLE IF NOT EXISTS `s2_view` ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + domain_id BIGINT, + `name` VARCHAR(255), + biz_name VARCHAR(255), + description VARCHAR(255), + status INT, + alias VARCHAR(255), + view_detail TEXT, + created_at TIMESTAMP, + created_by VARCHAR(255), + updated_at TIMESTAMP, + updated_by VARCHAR(255), + filter_sql VARCHAR(1000), + query_config VARCHAR(3000), + `admin` varchar(3000) DEFAULT NULL, + `admin_org` varchar(3000) DEFAULT NULL +); \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 0ca5c9542..ce2ef9633 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -1,4 +1,3 @@ --------demo for semantic and chat CREATE TABLE `s2_user_department` ( `user_name` varchar(200) NOT NULL, `department` varchar(200) NOT NULL @@ -336,7 +335,7 @@ CREATE TABLE `s2_model` ( CREATE TABLE `s2_plugin` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `type` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DASHBOARD,WIDGET,URL', - `model` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `view` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `pattern` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL, `parse_mode` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `parse_mode_config` text COLLATE utf8mb4_unicode_ci, @@ -354,6 +353,7 @@ CREATE TABLE `s2_query_stat_info` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, `trace_id` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询标识', `model_id` bigint(20) DEFAULT NULL, + `view_id` bigint(20) DEFAULT NULL, `user` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '执行sql的用户', `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `query_type` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询对应的场景', @@ -406,17 +406,18 @@ CREATE TABLE `s2_semantic_pasre_info` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='语义层sql解析信息表'; -CREATE TABLE `s2_view_info` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `domain_id` bigint(20) DEFAULT NULL, - `type` varchar(20) DEFAULT NULL COMMENT 'datasource、dimension、metric', - `config` text COMMENT 'config detail', - `created_at` datetime DEFAULT NULL, - `created_by` varchar(100) DEFAULT NULL, - `updated_at` datetime DEFAULT NULL, - `updated_by` varchar(100) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `s2_canvas` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `domain_id` bigint(20) DEFAULT NULL, + `type` varchar(20) DEFAULT NULL COMMENT 'datasource、dimension、metric', + `config` text COMMENT 'config detail', + `created_at` datetime DEFAULT NULL, + `created_by` varchar(100) DEFAULT NULL, + `updated_at` datetime DEFAULT NULL, + `updated_by` varchar(100) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8; create table s2_user ( @@ -481,4 +482,24 @@ CREATE TABLE `s2_app` `updated_at` datetime null, `created_by` varchar(255) null, `updated_by` varchar(255) null +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE s2_view +( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + domain_id BIGINT, + `name` VARCHAR(255), + biz_name VARCHAR(255), + `description` VARCHAR(255), + `status` INT, + alias VARCHAR(255), + view_detail text, + created_at datetime, + created_by VARCHAR(255), + updated_at datetime, + updated_by VARCHAR(255), + filter_sql VARCHAR(1000), + query_config VARCHAR(3000), + `admin` varchar(3000) DEFAULT NULL, + `admin_org` varchar(3000) DEFAULT NULL )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java index 5215a2e4f..a80203908 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.chat; -import static org.junit.Assert.assertEquals; - import com.tencent.supersonic.BaseApplication; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; @@ -16,13 +14,16 @@ import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.service.QueryService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.mock.mockito.MockBean; + import java.time.LocalDate; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.mock.mockito.MockBean; + +import static org.junit.Assert.assertEquals; public class BaseTest extends BaseApplication { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java deleted file mode 100644 index 59c3c8a79..000000000 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.chat; - -import com.tencent.supersonic.BaseApplication; -import com.tencent.supersonic.chat.core.query.llm.analytics.LLMAnswerResp; -import com.tencent.supersonic.chat.server.service.AgentService; -import com.tencent.supersonic.common.config.EmbeddingConfig; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.mock.mockito.MockBean; - -public class MetricInterpretTest extends BaseApplication { - - @MockBean - private AgentService agentService; - @MockBean - private EmbeddingConfig embeddingConfig; - - @Test - public void testMetricInterpret() throws Exception { - MockConfiguration.mockMetricAgent(agentService); - MockConfiguration.mockEmbeddingUrl(embeddingConfig); - - LLMAnswerResp lLmAnswerResp = new LLMAnswerResp(); - lLmAnswerResp.setAssistantMessage("alice最近在超音数的访问情况有增多"); - - } - -} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java index 15ee9fdb5..7e2b95087 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.chat; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; - import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; @@ -15,13 +12,17 @@ import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.jupiter.api.Test; + +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; public class MetricTest extends BaseTest { @@ -150,7 +151,6 @@ public class MetricTest extends BaseTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("人均访问次数")); - expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问用户数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY")); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java index 6159ed2d3..b1a3421a2 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java @@ -1,18 +1,18 @@ package com.tencent.supersonic.chat; -import static org.mockito.Mockito.when; - import com.google.common.collect.Lists; -import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.chat.core.plugin.PluginManager; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.embedding.Retrieval; import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; +import com.tencent.supersonic.util.DataUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; +import static org.mockito.Mockito.when; + @Configuration @Slf4j public class MockConfiguration { diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java index c80a755f4..35e0afd68 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.chat; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; - import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -11,11 +9,13 @@ import com.tencent.supersonic.chat.core.query.rule.tag.TagFilterQuery; import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; +import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.List; -import org.junit.jupiter.api.Test; public class TagTest extends BaseTest { @@ -29,7 +29,7 @@ public class TagTest extends BaseTest { expectedResult.setChatContext(expectedParseInfo); expectedResult.setQueryMode(MetricTagQuery.QUERY_MODE); - expectedParseInfo.setAggType(NONE); + expectedParseInfo.setAggType(AggregateTypeEnum.NONE); QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 7L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); @@ -53,7 +53,7 @@ public class TagTest extends BaseTest { expectedResult.setChatContext(expectedParseInfo); expectedResult.setQueryMode(TagFilterQuery.QUERY_MODE); - expectedParseInfo.setAggType(NONE); + expectedParseInfo.setAggType(AggregateTypeEnum.NONE); List list = new ArrayList<>(); list.add("流行"); @@ -73,7 +73,7 @@ public class TagTest extends BaseTest { expectedParseInfo.getDimensions().add(dim3); expectedParseInfo.getDimensions().add(dim4); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay, 7)); expectedParseInfo.setQueryType(QueryType.TAG); assertQueryResult(expectedResult, actualResult); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java index e7be59645..e87f7d513 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.chat.mapper; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; - import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -15,6 +13,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import org.junit.jupiter.api.Test; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; + public class MapperTest extends BaseTest { @Test diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java deleted file mode 100644 index 1f01ddf23..000000000 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.tencent.supersonic.chat.model; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.BaseApplication; -import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import com.tencent.supersonic.headless.server.service.MetricService; -import java.util.List; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; - -public class MetricServiceImplTest extends BaseApplication { - - @Autowired - protected MetricService metricService; - - @Test - void getMetrics() { - MetricFilter metricFilter = new MetricFilter(); - metricFilter.setType(MetricType.ATOMIC.name()); - metricFilter.setModelIds(Lists.newArrayList(1L)); - metricFilter.setSensitiveLevel(SensitiveLevelEnum.LOW.ordinal()); - List metricResps = metricService.getMetrics(metricFilter); - Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> metricResp.getModelId().equals(2L))); - Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> - metricResp.getSensitiveLevel().equals(SensitiveLevelEnum.HIGH.ordinal()))); - } - -} \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java index be8f54bf1..af28a96dd 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java @@ -1,9 +1,6 @@ package com.tencent.supersonic.headless; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.headless.api.pojo.enums.QueryType; @@ -12,9 +9,13 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.util.DataUtils; -import java.util.Arrays; import org.junit.jupiter.api.Test; +import java.util.Arrays; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + public class ExplainTest extends BaseTest { @Test @@ -22,7 +23,7 @@ public class ExplainTest extends BaseTest { String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "; ExplainSqlReq explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryType.SQL) - .queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentIModelIds())) + .queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView())) .build(); ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser()); assertNotNull(explain); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index bb658b7f1..4a0d656ad 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -1,7 +1,5 @@ package com.tencent.supersonic.util; -import static java.time.LocalDate.now; - import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -18,6 +16,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import java.util.HashSet; import java.util.Set; +import static java.time.LocalDate.now; + public class DataUtils { public static final Integer metricAgentId = 1; @@ -92,6 +92,15 @@ public class DataUtils { return dateInfo; } + public static DateConf getDateConf(DateConf.DateMode dateMode, String startDate, String endDate, int unit) { + DateConf dateInfo = new DateConf(); + dateInfo.setDateMode(dateMode); + dateInfo.setStartDate(startDate); + dateInfo.setEndDate(endDate); + dateInfo.setUnit(unit); + return dateInfo; + } + public static Agent getMetricAgent() { Agent agent = new Agent(); agent.setId(1); @@ -117,7 +126,7 @@ public class DataUtils { private static RuleParserTool getRuleQueryTool() { RuleParserTool ruleQueryTool = new RuleParserTool(); ruleQueryTool.setType(AgentToolType.NL2SQL_RULE); - ruleQueryTool.setModelIds(Lists.newArrayList(-1L)); + ruleQueryTool.setViewIds(Lists.newArrayList(-1L)); ruleQueryTool.setQueryModes(Lists.newArrayList("METRIC_TAG", "METRIC_FILTER", "METRIC_MODEL", "TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID")); return ruleQueryTool; @@ -137,4 +146,8 @@ public class DataUtils { result.add(3L); return result; } + + public static Long getMetricAgentView() { + return 1L; + } } diff --git a/launchers/standalone/src/test/resources/META-INF/spring.factories b/launchers/standalone/src/test/resources/META-INF/spring.factories index 7e11c81ce..80b5b0431 100644 --- a/launchers/standalone/src/test/resources/META-INF/spring.factories +++ b/launchers/standalone/src/test/resources/META-INF/spring.factories @@ -2,8 +2,7 @@ com.tencent.supersonic.chat.core.mapper.SchemaMapper=\ com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \ com.tencent.supersonic.chat.core.mapper.KeywordMapper, \ com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.core.mapper.EntityMapper, \ - com.tencent.supersonic.chat.core.mapper.ModelClusterMapper + com.tencent.supersonic.chat.core.mapper.EntityMapper com.tencent.supersonic.chat.core.parser.SemanticParser=\ com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \ @@ -17,8 +16,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.corrector.SelectCorrector, \ com.tencent.supersonic.chat.core.corrector.WhereCorrector, \ com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \ - com.tencent.supersonic.chat.core.corrector.HavingCorrector, \ - com.tencent.supersonic.chat.core.corrector.FromCorrector + com.tencent.supersonic.chat.core.corrector.HavingCorrector com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.server.processor.parse.MetricCheckProcessor, \ @@ -33,8 +31,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\ com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\ com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter -com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\ - com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver +com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\ + com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt new file mode 100644 index 000000000..7bc20ae1d --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt @@ -0,0 +1,8 @@ +阿里云 _10_20 5 +天猫 _10_20 5 +腾讯游戏 _10_20 5 +度小满 _10_20 5 +京东金融 _10_20 5 + + + diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt new file mode 100644 index 000000000..4d9cccf5a --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt @@ -0,0 +1,8 @@ +张勇 _10_22 5 +马化腾 _10_22 5 +朱光 _10_22 5 +刘强东 _10_22 5 + + + + diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt index b01cf97ae..9871036d4 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _4_7 100 -陈奕迅 _4_7 100 -林俊杰 _4_7 100 -张碧晨 _4_7 100 -程响 _4_7 100 -Taylor#Swift _4_7 100 -内地 _4_4 100 -欧美 _4_4 100 -港台 _4_4 100 -流行 _4_6 100 -国风 _4_6 100 \ No newline at end of file +周杰伦 _2_7 100 +陈奕迅 _2_7 100 +林俊杰 _2_7 100 +张碧晨 _2_7 100 +程响 _2_7 100 +Taylor#Swift _2_7 100 +内地 _2_4 100 +欧美 _2_4 100 +港台 _2_4 100 +流行 _2_6 100 +国风 _2_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt index e442a219c..d0d411751 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _3_3 52 -p2 _3_3 47 -p3 _3_3 31 -p4 _3_3 36 -p5 _3_3 50 -p6 _3_3 38 \ No newline at end of file +p1 _2_3 52 +p2 _2_3 47 +p3 _2_3 31 +p4 _2_3 36 +p5 _2_3 50 +p6 _2_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt index 3bc23d613..6699a0fc8 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _4_7 9000 -周深 _4_7 8000 -周传雄 _4_7 7000 -周华建 _4_7 6000 -陈奕迅 _4_7 8000 -林俊杰 _4_7 7000 -张碧晨 _4_7 7000 -程响 _4_7 7000 -Taylor#Swift _4_7 7000 \ No newline at end of file +周杰伦 _2_7 9000 +周深 _2_7 8000 +周传雄 _2_7 7000 +周华建 _2_7 6000 +陈奕迅 _2_7 8000 +林俊杰 _2_7 7000 +张碧晨 _2_7 7000 +程响 _2_7 7000 +Taylor#Swift _2_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt new file mode 100644 index 000000000..5a2cb8f1d --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt @@ -0,0 +1,5 @@ +百度集团 _9_15 5 +阿里巴巴集团 _9_15 5 +深圳市腾讯计算机系统有限公司 _9_15 5 +北京京东世纪贸易有限公司 _9_15 5 +网易公司 _9_15 5 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt new file mode 100644 index 000000000..c4e7f41d6 --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt @@ -0,0 +1,4 @@ +北京 _9_16 5 +杭州 _9_16 5 +深圳 _9_16 5 + diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt new file mode 100644 index 000000000..b15cc33d9 --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt @@ -0,0 +1,7 @@ +李彦宏 _9_18 5 +马云 _9_18 5 +马化腾 _9_18 5 +刘强东 _9_18 5 +丁磊 _9_18 5 + + diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt new file mode 100644 index 000000000..233cfe17f --- /dev/null +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt @@ -0,0 +1,7 @@ +李彦宏 _9_19 5 +张勇 _9_19 5 +刘炽平 _9_19 5 +刘强东 _9_19 5 +丁磊 _9_19 5 + + diff --git a/launchers/standalone/src/test/resources/db/data-h2.sql b/launchers/standalone/src/test/resources/db/data-h2.sql index 5096c1997..860fb7264 100644 --- a/launchers/standalone/src/test/resources/db/data-h2.sql +++ b/launchers/standalone/src/test/resources/db/data-h2.sql @@ -14,7 +14,7 @@ MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`sta values (3 , 3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -MERGE INTO s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) +MERGE INTO s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` ) values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin'); -- sample data diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 273ad467f..787ebaefc 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -230,7 +230,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela PRIMARY KEY (`id`) ); -create table IF NOT EXISTS s2_view_info +create table IF NOT EXISTS s2_canvas ( id INT auto_increment, domain_id INT null, @@ -242,13 +242,14 @@ create table IF NOT EXISTS s2_view_info updated_by varchar(100) not null, PRIMARY KEY (`id`) ); -COMMENT ON TABLE s2_view_info IS 'view information table'; +COMMENT ON TABLE s2_canvas IS 'view information table'; CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( `id` INT NOT NULL AUTO_INCREMENT, `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier `model_id` INT DEFAULT NULL, + `view_id` INT DEFAULT NULL, `user` varchar(200) DEFAULT NULL, `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene @@ -326,7 +327,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` ( `id` INT AUTO_INCREMENT, `type` varchar(50) NULL, - `model` varchar(100) NULL, + `view` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, `parse_mode_config` LONGVARCHAR NULL, @@ -498,4 +499,23 @@ CREATE TABLE IF NOT EXISTS `s2_app` ( created_by VARCHAR(255), updated_at TIMESTAMP, updated_by VARCHAR(255) +); + +CREATE TABLE IF NOT EXISTS `s2_view` ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + domain_id BIGINT, + `name` VARCHAR(255), + biz_name VARCHAR(255), + description VARCHAR(255), + status INT, + alias VARCHAR(255), + view_detail TEXT, + created_at TIMESTAMP, + created_by VARCHAR(255), + updated_at TIMESTAMP, + updated_by VARCHAR(255), + filter_sql VARCHAR(1000), + query_config VARCHAR(3000), + `admin` varchar(3000) DEFAULT NULL, + `admin_org` varchar(3000) DEFAULT NULL ); \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1a28580ab..a6f91c0ec 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 3.1.2 0.4.6 1.4.200 - 2.0.24 + 2.0.40 6.5.0 4.5.6 2.12.1