From 27bb1b322e5970bf84796f116886b48cb4b89982 Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:05:24 +0800 Subject: [PATCH] (improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423) Co-authored-by: jolunoluo --- .../request/QueryAuthResReq.java | 20 +- .../application/AuthServiceImpl.java | 8 +- .../api/component/SemanticInterpreter.java | 2 +- .../supersonic/chat/api/pojo/ModelSchema.java | 19 +- .../chat/api/pojo/QueryContext.java | 1 + .../chat/api/pojo/SchemaElementType.java | 4 +- .../api/pojo/SchemaModelClusterMapInfo.java | 61 + .../chat/api/pojo/SemanticParseInfo.java | 42 +- .../chat/api/pojo/SemanticSchema.java | 135 +- .../chat/api/pojo/request/QueryReq.java | 2 +- .../chat/api/pojo/request/SolvedQueryReq.java | 2 +- .../chat/api/pojo/response/ModelInfo.java | 6 +- .../chat/corrector/BaseSemanticCorrector.java | 19 +- .../chat/corrector/FromCorrector.java | 17 + .../chat/corrector/GroupByCorrector.java | 9 +- .../chat/corrector/HavingCorrector.java | 11 +- .../chat/corrector/SchemaCorrector.java | 2 +- .../chat/corrector/WhereCorrector.java | 18 +- .../chat/llm/EmbedLLMInterpreter.java | 5 +- .../chat/llm/HttpLLMInterpreter.java | 8 +- .../supersonic/chat/llm/LLMInterpreter.java | 2 +- .../supersonic/chat/mapper/EntityMapper.java | 8 +- .../chat/mapper/ModelClusterMapper.java | 52 + .../chat/parser/QueryTypeParser.java | 21 +- .../llm/interpret/MetricInterpretParser.java | 19 +- .../llm/s2sql/HeuristicModelResolver.java | 160 +- .../parser/llm/s2sql/LLMRequestService.java | 91 +- .../parser/llm/s2sql/LLMResponseService.java | 26 +- .../chat/parser/llm/s2sql/LLMS2SQLParser.java | 26 +- .../chat/parser/llm/s2sql/ModelResolver.java | 2 +- .../chat/parser/llm/s2sql/ParseResult.java | 6 +- .../chat/parser/plugin/PluginParser.java | 16 +- .../chat/parser/rule/AgentCheckParser.java | 8 +- .../parser/rule/ContextInheritParser.java | 84 +- .../chat/parser/rule/QueryModeParser.java | 15 +- .../MetricCheckPostProcessor.java | 69 +- .../ParseInfoUpdateProcessor.java | 193 ++- .../chat/query/BaseSemanticQuery.java | 13 +- .../supersonic/chat/query/QueryRanker.java | 72 - .../chat/query/llm/s2sql/S2SQLQuery.java | 7 +- .../query/plugin/webpage/WebPageQuery.java | 11 +- .../chat/query/rule/RuleSemanticQuery.java | 47 +- .../rule/metric/MetricSemanticQuery.java | 16 +- .../query/rule/metric/MetricTagQuery.java | 2 +- .../query/rule/metric/MetricTopNQuery.java | 18 +- .../chat/query/rule/tag/TagListQuery.java | 17 +- .../chat/query/rule/tag/TagSemanticQuery.java | 19 +- .../execute/EntityInfoExecuteResponder.java | 2 +- .../SimilarMetricExecuteResponder.java | 2 +- .../parse/QueryRankParseResponder.java | 66 +- .../chat/rest/ChatConfigController.java | 11 +- .../supersonic/chat/service/ChatService.java | 3 +- .../chat/service/ParseInfoService.java | 9 - .../chat/service/SemanticService.java | 139 +- .../chat/service/impl/ChatServiceImpl.java | 27 +- .../chat/service/impl/ConfigServiceImpl.java | 41 +- .../service/impl/ParserInfoServiceImpl.java | 205 --- .../chat/service/impl/PluginServiceImpl.java | 15 +- .../chat/service/impl/QueryServiceImpl.java | 2 +- .../chat/service/impl/SearchServiceImpl.java | 31 +- .../supersonic/chat/utils/DictMetaHelper.java | 29 +- .../chat/utils/ModelClusterBuilder.java | 42 + .../chat/utils/QueryReqBuilder.java | 19 +- .../MetricCheckPostProcessorTest.java | 44 +- .../chat/test/context/ContextTest.java | 2 +- .../test/context/MockBeansConfiguration.java | 28 +- .../semantic/LocalSemanticInterpreter.java | 9 +- .../semantic/ModelSchemaBuilder.java | 32 +- .../semantic/RemoteSemanticInterpreter.java | 27 +- .../supersonic/common/pojo/JoinCondition.java | 19 + .../supersonic/common/pojo/ModelCluster.java | 52 + .../supersonic/common/pojo/ModelRela.java | 24 + .../common/pojo/enums/FilterOperatorEnum.java | 7 + .../jsqlparser/SqlParserSelectHelper.java | 15 +- .../com/tencent/supersonic/ConfigureDemo.java | 78 +- .../tencent/supersonic/LoadBenchMarkDemo.java | 135 +- .../tencent/supersonic/LoadModelDataDemo.java | 207 ++- .../main/resources/META-INF/spring.factories | 6 +- .../data/dictionary/custom/DimValue_1_2.txt | 22 +- .../data/dictionary/custom/DimValue_1_3.txt | 12 +- .../data/dictionary/custom/DimValue_2_7.txt | 9 - .../data/dictionary/custom/DimValue_4_9.txt | 9 + .../dictionary/custom/benchmark_cspider.txt | 68 - .../src/main/resources/db/schema-h2.sql | 38 +- .../src/main/resources/db/schema-mysql.sql | 10 + .../src/main/resources/db/sql-update.sql | 19 +- .../integration/MetricQueryTest.java | 65 +- .../test/resources/META-INF/spring.factories | 3 +- .../src/test/resources/db/schema-h2.sql | 142 +- ...DatasourceDetail.java => ModelDetail.java} | 12 +- .../api/model/request/DatasourceRelaReq.java | 23 - .../api/model/request/DatasourceReq.java | 56 - .../api/model/request/DimensionReq.java | 3 - .../semantic/api/model/request/DomainReq.java | 22 +- .../semantic/api/model/request/ModelReq.java | 31 +- .../model/response/DatasourceRelaResp.java | 27 - .../api/model/response/DatasourceResp.java | 20 - .../api/model/response/DimSchemaResp.java | 4 + .../api/model/response/DimensionResp.java | 8 +- .../api/model/response/MetricResp.java | 6 +- .../api/model/response/ModelResp.java | 57 +- .../model/response/ModelSchemaRelaResp.java | 5 +- .../api/model/response/ModelSchemaResp.java | 21 +- ...urceYamlTpl.java => DataModelYamlTpl.java} | 2 +- .../api/query/request/QueryS2SQLReq.java | 17 +- .../api/query/request/QueryStructReq.java | 34 +- .../MaterializationConfServiceImpl.java | 64 +- .../model/application/CatalogImpl.java | 91 +- .../application/DatabaseServiceImpl.java | 30 +- .../application/DatasourceServiceImpl.java | 355 ---- .../application/DimensionServiceImpl.java | 57 +- .../model/application/DomainServiceImpl.java | 52 +- .../model/application/MetricServiceImpl.java | 56 +- .../application/ModelRelaServiceImpl.java | 73 + .../model/application/ModelServiceImpl.java | 514 ++++-- .../application/ViewInfoServiceImpl.java | 41 +- .../semantic/model/domain/Catalog.java | 23 +- .../model/domain/DatasourceService.java | 42 - .../model/domain/ModelRelaService.java | 20 + .../semantic/model/domain/ModelService.java | 44 +- .../model/domain/dataobject/DatasourceDO.java | 78 - .../dataobject/DatasourceDOExample.java | 1015 ------------ .../model/domain/dataobject/DimensionDO.java | 63 +- .../domain/dataobject/DimensionDOExample.java | 1410 ---------------- .../model/domain/dataobject/DomainDO.java | 231 +-- .../domain/dataobject/MetricDOExample.java | 1435 ----------------- .../model/domain/dataobject/ModelDO.java | 379 +---- .../domain/dataobject/ModelDOExample.java | 1435 ----------------- .../model/domain/dataobject/ModelRelaDO.java | 26 + .../domain/manager/DatasourceYamlManager.java | 30 +- .../model/domain/pojo/Datasource.java | 4 +- .../model/domain/pojo/MetaFilter.java | 2 - .../model/domain/pojo/ModelFilter.java | 14 + .../repository/DatasourceRepository.java | 22 - .../domain/repository/ModelRepository.java | 8 +- .../domain/utils/DatasourceConverter.java | 201 --- .../domain/utils/DimensionConverter.java | 31 +- .../model/domain/utils/DomainConvert.java | 36 +- .../model/domain/utils/MetricConverter.java | 10 +- .../model/domain/utils/ModelConvert.java | 58 - .../model/domain/utils/ModelConverter.java | 218 +++ .../infrastructure/mapper/DomainDOMapper.java | 52 +- .../mapper/ModelDOCustomMapper.java | 14 + .../infrastructure/mapper/ModelDOMapper.java | 82 +- ...ceDOMapper.java => ModelRelaDOMapper.java} | 8 +- .../repository/DatasourceRepositoryImpl.java | 62 - .../repository/DomainRepositoryImpl.java | 15 +- .../repository/ModelRepositoryImpl.java | 60 +- .../model/rest/DatasourceController.java | 66 - .../semantic/model/rest/ModelController.java | 30 +- .../model/rest/ModelRelaController.java | 49 + .../model/rest/ViewInfoController.java | 16 +- .../main/resources/mapper/ModelDOMapper.xml | 480 ------ .../mapper/custom/DimensionDOCustomMapper.xml | 10 +- .../mapper/custom/ModelDOCustomMapper.xml | 18 + .../DatasourceServiceImplTest.java | 218 --- .../application/ModelServiceImplTest.java | 241 +++ .../query/optimizer/MaterializationQuery.java | 12 +- .../parser/calcite/CalciteSqlParser.java | 1 + .../parser/calcite/SemanticSchemaManager.java | 51 +- .../parser/calcite/planner/AggPlanner.java | 29 +- .../parser/calcite/s2sql/JoinRelation.java | 17 + .../parser/calcite/s2sql/SemanticModel.java | 1 + .../parser/calcite/schema/SemanticSchema.java | 18 +- .../calcite/sql/node/DataSourceNode.java | 123 +- .../parser/calcite/sql/node/SemanticNode.java | 62 +- .../parser/calcite/sql/render/JoinRender.java | 116 +- .../parser/convert/CalculateAggConverter.java | 19 +- .../convert/DefaultDimValueConverter.java | 2 +- .../parser/convert/MetricCheckConverter.java | 5 +- .../query/parser/convert/MultiSourceJoin.java | 135 -- .../convert/ParserDefaultConverter.java | 12 +- .../parser/convert/QueryReqConverter.java | 84 +- .../persistence/pojo/QueryStatement.java | 5 +- .../semantic/query/rest/SchemaController.java | 23 +- .../query/service/AuthCommonService.java | 60 +- .../service/MaterializationServiceImpl.java | 17 +- .../query/service/QueryServiceImpl.java | 50 +- .../semantic/query/service/SchemaService.java | 2 +- .../query/service/SchemaServiceImpl.java | 25 +- .../service/SemanticQueryEngineImpl.java | 7 +- .../query/utils/ComponentFactory.java | 2 - .../query/utils/DataPermissionAOP.java | 315 +--- .../semantic/query/utils/DimValueAspect.java | 25 +- .../query/utils/QueryStructUtils.java | 53 +- .../semantic/query/utils/QueryUtils.java | 27 +- .../semantic/query/utils/S2SQLDataAspect.java | 48 +- .../semantic/query/utils/StatUtils.java | 31 +- .../calcite/MaterializationQueryTest.java | 5 +- .../calcite/SemanticParserServiceTest.java | 6 +- 190 files changed, 3900 insertions(+), 10561 deletions(-) create mode 100644 chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FromCorrector.java create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/mapper/ModelClusterMapper.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryRanker.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/service/ParseInfoService.java delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/utils/ModelClusterBuilder.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java delete mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_2_7.txt create mode 100644 launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt rename semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/{DatasourceDetail.java => ModelDetail.java} (52%) delete mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceRelaReq.java delete mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java delete mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceRelaResp.java delete mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java rename semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/{DatasourceYamlTpl.java => DataModelYamlTpl.java} (91%) delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelRelaServiceImpl.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelRelaService.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDOExample.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/MetricDOExample.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelRelaDO.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/ModelFilter.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOCustomMapper.java rename semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/{DatasourceDOMapper.java => ModelRelaDOMapper.java} (54%) delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java delete mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelRelaController.java delete mode 100644 semantic/model/src/main/resources/mapper/ModelDOMapper.xml create mode 100644 semantic/model/src/main/resources/mapper/custom/ModelDOCustomMapper.xml delete mode 100644 semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImplTest.java create mode 100644 semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/ModelServiceImplTest.java create mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/JoinRelation.java delete mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java index c1dc18207..250b63689 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.auth.api.authorization.request; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; -import java.util.ArrayList; -import java.util.List; - import lombok.Data; import lombok.ToString; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; @Data @ToString @@ -16,4 +18,16 @@ public class QueryAuthResReq { private List resources; private Long modelId; + + private List modelIds; + + public List getModelIds() { + if (!CollectionUtils.isEmpty(modelIds)) { + return modelIds; + } + if (modelId != null) { + return Lists.newArrayList(modelId); + } + return Lists.newArrayList(); + } } diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java index b3ba28775..f0695ce23 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java @@ -47,7 +47,7 @@ public class AuthServiceImpl implements AuthService { public List queryAuthGroups(String modelId, Integer groupId) { return load().stream() .filter(group -> (Objects.isNull(groupId) || groupId.equals(group.getGroupId())) - && modelId.equals(group.getModelId())) + && modelId.equals(group.getModelId().toString())) .collect(Collectors.toList()); } @@ -79,7 +79,7 @@ public class AuthServiceImpl implements AuthService { @Override public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) { Set userOrgIds = userService.getUserAllOrgId(user.getName()); - List groups = getAuthGroups(req.getModelId(), user.getName(), new ArrayList<>(userOrgIds)); + List groups = getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds)); AuthorizedResourceResp resource = new AuthorizedResourceResp(); Map> authGroupsByModelId = groups.stream() .collect(Collectors.groupingBy(AuthGroup::getModelId)); @@ -126,10 +126,10 @@ public class AuthServiceImpl implements AuthService { return resource; } - private List getAuthGroups(Long modelId, String userName, List departmentIds) { + private List getAuthGroups(List modelIds, String userName, List departmentIds) { List groups = load().stream() .filter(group -> { - if (modelId != null && Objects.equals(group.getModelId(), modelId)) { + if (CollectionUtils.isEmpty(modelIds) || !modelIds.contains(group.getModelId())) { return false; } if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers() diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java index c1cf4c5e9..8105cb560 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java @@ -9,8 +9,8 @@ import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java index de0f8b1dd..bb74c269a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java @@ -1,8 +1,13 @@ package com.tencent.supersonic.chat.api.pojo; +import com.google.common.collect.Sets; +import com.tencent.supersonic.common.pojo.ModelRela; import lombok.Data; +import org.springframework.util.CollectionUtils; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -14,8 +19,8 @@ public class ModelSchema { private Set dimensions = new HashSet<>(); private Set dimensionValues = new HashSet<>(); private Set tags = new HashSet<>(); - @Deprecated private SchemaElement entity = new SchemaElement(); + private List modelRelas = new ArrayList<>(); public SchemaElement getElement(SchemaElementType elementType, long elementID) { Optional element = Optional.empty(); @@ -75,4 +80,16 @@ public class ModelSchema { } } + public Set getModelClusterSet() { + if (CollectionUtils.isEmpty(modelRelas)) { + return Sets.newHashSet(); + } + Set modelClusterSet = new HashSet<>(); + modelRelas.forEach(modelRela -> { + modelClusterSet.add(modelRela.getToModelId()); + modelClusterSet.add(modelRela.getFromModelId()); + }); + return modelClusterSet; + } + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java index 91ccef717..3a6c30991 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java @@ -15,6 +15,7 @@ public class QueryContext { private QueryReq request; private List candidateQueries = new ArrayList<>(); private SchemaMapInfo mapInfo = new SchemaMapInfo(); + private SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo(); public QueryContext(QueryReq request) { this.request = request; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java index 891769c07..b5ae5b6b1 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementType.java @@ -6,7 +6,7 @@ public enum SchemaElementType { DIMENSION, VALUE, ENTITY, + TAG, ID, - DATE, - TAG + DATE } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java new file mode 100644 index 000000000..43723e40f --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaModelClusterMapInfo.java @@ -0,0 +1,61 @@ +package com.tencent.supersonic.chat.api.pojo; + +import com.clickhouse.client.internal.apache.commons.compress.utils.Lists; +import com.tencent.supersonic.common.pojo.ModelCluster; +import lombok.Data; +import org.springframework.util.CollectionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Data +public class SchemaModelClusterMapInfo { + + private Map> modelElementMatches = new HashMap<>(); + + public Set getMatchedModelClusters() { + return modelElementMatches.keySet(); + } + + public List getMatchedElements(Long modelId) { + for (String key : modelElementMatches.keySet()) { + if (ModelCluster.getModelIdFromKey(key).contains(modelId)) { + return modelElementMatches.get(key); + } + } + return Lists.newArrayList(); + } + + public List getMatchedElements(String modelCluster) { + return modelElementMatches.get(modelCluster); + } + + public Map> getModelElementMatches() { + return modelElementMatches; + } + + public Map> getElementMatchesByModelIds(Set modelIds) { + if (CollectionUtils.isEmpty(modelIds)) { + return modelElementMatches; + } + Map> modelElementMatchesFiltered = new HashMap<>(); + for (String key : modelElementMatches.keySet()) { + for (Long modelId : modelIds) { + if (ModelCluster.getModelIdFromKey(key).contains(modelId)) { + modelElementMatchesFiltered.put(key, modelElementMatches.get(key)); + } + } + } + return modelElementMatchesFiltered; + } + + public void setModelElementMatches(Map> modelElementMatches) { + this.modelElementMatches = modelElementMatches; + } + + public void setMatchedElements(String modelCluster, List elementMatches) { + modelElementMatches.put(modelCluster, elementMatches); + } +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 221897edc..f5fcf046c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -5,9 +5,13 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.common.pojo.enums.FilterType; +import lombok.Data; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -16,15 +20,13 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; -import com.tencent.supersonic.common.pojo.enums.FilterType; -import lombok.Data; @Data public class SemanticParseInfo { private Integer id; private String queryMode; - private SchemaElement model; + private ModelCluster model = new ModelCluster(); private Set metrics = new TreeSet<>(new SchemaNameLengthComparator()); private Set dimensions = new LinkedHashSet(); private SchemaElement entity; @@ -42,12 +44,18 @@ public class SemanticParseInfo { private SqlInfo sqlInfo = new SqlInfo(); private QueryType queryType = QueryType.OTHER; - public Long getModelId() { - return model != null ? model.getId() : 0L; + public String getModelClusterKey() { + if (model == null) { + return ""; + } + return model.getKey(); } public String getModelName() { - return model != null ? model.getName() : "null"; + if (model == null) { + return ""; + } + return model.getName(); } private static class SchemaNameLengthComparator implements Comparator { @@ -78,4 +86,26 @@ public class SemanticParseInfo { return metrics; } + private Map getModelElementCountMap() { + Map elementCountMap = new HashMap<>(); + elementMatches.forEach(element -> { + int count = elementCountMap.getOrDefault(element.getElement().getModel(), 0); + elementCountMap.put(element.getElement().getModel(), count + 1); + }); + return elementCountMap; + } + + public Long getModelId() { + Map elementCountMap = getModelElementCountMap(); + Long modelId = -1L; + int maxCnt = 0; + for (Long model : elementCountMap.keySet()) { + if (elementCountMap.get(model) > maxCnt) { + maxCnt = elementCountMap.get(model); + modelId = model; + } + } + return modelId; + } + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 1f3b01e85..fbf4c5a7c 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -1,9 +1,14 @@ package com.tencent.supersonic.chat.api.pojo; +import org.springframework.util.CollectionUtils; + import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; public class SemanticSchema implements Serializable { @@ -18,6 +23,64 @@ public class SemanticSchema implements Serializable { modelSchemaList.add(schema); } + public SchemaElement getElement(SchemaElementType elementType, long elementID) { + Optional element = Optional.empty(); + + switch (elementType) { + case ENTITY: + element = getElementsById(elementID, getEntities()); + break; + case MODEL: + element = getElementsById(elementID, getModels()); + break; + case METRIC: + element = getElementsById(elementID, getMetrics()); + break; + case DIMENSION: + element = getElementsById(elementID, getDimensions()); + break; + case VALUE: + element = getElementsById(elementID, getDimensionValues()); + break; + default: + } + + if (element.isPresent()) { + return element.get(); + } else { + return null; + } + } + + public SchemaElement getElementByName(SchemaElementType elementType, String name) { + Optional element = Optional.empty(); + + switch (elementType) { + case ENTITY: + element = getElementsByName(name, getEntities()); + break; + case MODEL: + element = getElementsByName(name, getModels()); + break; + case METRIC: + element = getElementsByName(name, getMetrics()); + break; + case DIMENSION: + element = getElementsByName(name, getDimensions()); + break; + case VALUE: + element = getElementsByName(name, getDimensionValues()); + break; + default: + } + + if (element.isPresent()) { + return element.get(); + } else { + return null; + } + } + public Map getModelIdToName() { return modelSchemaList.stream() .collect(Collectors.toMap(a -> a.getModel().getId(), a -> a.getModel().getName(), (k1, k2) -> k1)); @@ -35,9 +98,21 @@ public class SemanticSchema implements Serializable { return dimensions; } - public List getDimensions(Long modelId) { + public List getDimensions(Set modelIds) { List dimensions = getDimensions(); - return getElementsByModelId(modelId, dimensions); + return getElementsByModelId(modelIds, dimensions); + } + + public SchemaElement getDimensions(Long id) { + List dimensions = getDimensions(); + Optional dimension = getElementsById(id, dimensions); + return dimension.orElse(null); + } + + public List getTags() { + List tags = new ArrayList<>(); + modelSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); + return tags; } public List getMetrics() { @@ -46,21 +121,9 @@ public class SemanticSchema implements Serializable { return metrics; } - public List getMetrics(Long modelId) { + public List getMetrics(Set modelIds) { List metrics = getMetrics(); - return getElementsByModelId(modelId, metrics); - } - - private List getElementsByModelId(Long modelId, List elements) { - return elements.stream() - .filter(schemaElement -> modelId.equals(schemaElement.getModel())) - .collect(Collectors.toList()); - } - - public List getModels() { - List models = new ArrayList<>(); - modelSchemaList.stream().forEach(d -> models.add(d.getModel())); - return models; + return getElementsByModelId(modelIds, metrics); } public List getEntities() { @@ -69,11 +132,43 @@ public class SemanticSchema implements Serializable { return entities; } - public Map getBizNameToName(Long modelId) { + private List getElementsByModelId(Set modelIds, List elements) { + return elements.stream() + .filter(schemaElement -> modelIds.contains(schemaElement.getModel())) + .collect(Collectors.toList()); + } + + private Optional getElementsById(Long id, List elements) { + return elements.stream() + .filter(schemaElement -> id.equals(schemaElement.getId())) + .findFirst(); + } + + private Optional getElementsByName(String name, List elements) { + return elements.stream() + .filter(schemaElement -> name.equals(schemaElement.getName())) + .findFirst(); + } + + public List getModels() { + List models = new ArrayList<>(); + modelSchemaList.stream().forEach(d -> models.add(d.getModel())); + return models; + } + + public Map getBizNameToName(Set modelIds) { List allElements = new ArrayList<>(); - allElements.addAll(getDimensions(modelId)); - allElements.addAll(getMetrics(modelId)); + allElements.addAll(getDimensions(modelIds)); + allElements.addAll(getMetrics(modelIds)); return allElements.stream() - .collect(Collectors.toMap(a -> a.getBizName(), a -> a.getName(), (k1, k2) -> k1)); + .collect(Collectors.toMap(SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1)); + } + + public Map getModelSchemaMap() { + if (CollectionUtils.isEmpty(modelSchemaList)) { + return new HashMap<>(); + } + return modelSchemaList.stream().collect(Collectors.toMap(modelSchema + -> modelSchema.getModel().getModel(), modelSchema -> modelSchema)); } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java index 04140c180..1d457610a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java @@ -7,7 +7,7 @@ import lombok.Data; public class QueryReq { private String queryText; private Integer chatId; - private Long modelId = 0L; + private Long modelId; private User user; private QueryFilters queryFilters; private boolean saveAnswer = true; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java index 1a59ec9b4..56506b48a 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/SolvedQueryReq.java @@ -18,7 +18,7 @@ public class SolvedQueryReq { private String queryText; - private Long modelId; + private String modelId; private Integer agentId; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java index b46c7b533..5944e7f5f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ModelInfo.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.chat.api.pojo.response; +import lombok.Data; + import java.io.Serializable; import java.util.List; -import lombok.Data; @Data public class ModelInfo extends DataInfo implements Serializable { private List words; - private String primaryEntityName; - private String primaryEntityBizName; + private String primaryKey; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java index ff034d8d9..e67399b50 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java @@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -19,9 +23,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; /** * basic semantic correction functionality, offering common methods and an @@ -45,7 +46,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { public abstract void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo); - protected Map getFieldNameMap(Long modelId) { + protected Map getFieldNameMap(Set modelIds) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); @@ -55,7 +56,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { // support fieldName and field alias Map result = dbAllFields.stream() - .filter(entry -> entry.getModel().equals(modelId)) + .filter(entry -> modelIds.contains(entry.getModel())) .flatMap(schemaElement -> { Set elements = new HashSet<>(); elements.add(schemaElement.getName()); @@ -103,9 +104,9 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) { //add aggregate to all metric String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - Long modelId = semanticParseInfo.getModel().getModel(); + Set modelIds = semanticParseInfo.getModel().getModelIds(); - List metrics = getMetricElements(modelId); + List metrics = getMetricElements(modelIds); Map metricToAggregate = metrics.stream() .map(schemaElement -> { @@ -122,9 +123,9 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql); } - protected List getMetricElements(Long modelId) { + protected List getMetricElements(Set modelIds) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - return semanticSchema.getMetrics(modelId); + return semanticSchema.getMetrics(modelIds); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FromCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FromCorrector.java new file mode 100644 index 000000000..cb8557f6f --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FromCorrector.java @@ -0,0 +1,17 @@ +package com.tencent.supersonic.chat.corrector; + +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FromCorrector extends BaseSemanticCorrector { + + @Override + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + String modelName = semanticParseInfo.getModel().getName(); + SqlParserReplaceHelper.replaceTable(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), modelName); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java index 5d6a8a7b4..47e25ead4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java @@ -9,12 +9,13 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; /** * Perform SQL corrections on the "group by" section in S2SQL. @@ -30,14 +31,14 @@ public class GroupByCorrector extends BaseSemanticCorrector { } private void addGroupByFields(SemanticParseInfo semanticParseInfo) { - Long modelId = semanticParseInfo.getModel().getModel(); + Set modelIds = semanticParseInfo.getModel().getModelIds(); //add dimension group by SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String correctS2SQL = sqlInfo.getCorrectS2SQL(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); //add alias field name - Set dimensions = semanticSchema.getDimensions(modelId).stream() + Set dimensions = semanticSchema.getDimensions(modelIds).stream() .flatMap( schemaElement -> { Set elements = new HashSet<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java index 826e6b942..a8530fc38 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java @@ -10,13 +10,14 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import org.springframework.util.CollectionUtils; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + /** * Perform SQL corrections on the "Having" section in S2SQL. */ @@ -37,11 +38,11 @@ public class HavingCorrector extends BaseSemanticCorrector { } private void addHaving(SemanticParseInfo semanticParseInfo) { - Long modelId = semanticParseInfo.getModel().getModel(); + Set modelIds = semanticParseInfo.getModel().getModelIds(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - Set metrics = semanticSchema.getMetrics(modelId).stream() + Set metrics = semanticSchema.getMetrics(modelIds).stream() .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); if (CollectionUtils.isEmpty(metrics)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java index 7eb6ed8fa..b164ab2ec 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java @@ -51,7 +51,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { } private void correctFieldName(SemanticParseInfo semanticParseInfo) { - Map fieldNameMap = getFieldNameMap(semanticParseInfo.getModelId()); + Map fieldNameMap = getFieldNameMap(semanticParseInfo.getModel().getModelIds()); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap); sqlInfo.setCorrectS2SQL(sql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java index de50ec147..839cbc03b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java @@ -16,11 +16,6 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -29,6 +24,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + /** * Perform SQL corrections on the "Where" section in S2SQL. */ @@ -102,10 +104,8 @@ public class WhereCorrector extends BaseSemanticCorrector { private void updateFieldValueByTechName(SemanticParseInfo semanticParseInfo) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - Long modelId = semanticParseInfo.getModel().getId(); - List dimensions = semanticSchema.getDimensions().stream() - .filter(schemaElement -> modelId.equals(schemaElement.getModel())) - .collect(Collectors.toList()); + Set modelIds = semanticParseInfo.getModel().getModelIds(); + List dimensions = semanticSchema.getDimensions(modelIds); if (CollectionUtils.isEmpty(dimensions)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/EmbedLLMInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/EmbedLLMInterpreter.java index bb5130a1e..19236c6a1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/EmbedLLMInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/EmbedLLMInterpreter.java @@ -17,15 +17,16 @@ import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; +import lombok.extern.slf4j.Slf4j; + import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; @Slf4j public class EmbedLLMInterpreter implements LLMInterpreter { - public LLMResp query2sql(LLMReq llmReq, Long modelId) { + public LLMResp query2sql(LLMReq llmReq, String modelClusterKey) { ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/HttpLLMInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/HttpLLMInterpreter.java index 9613bcdea..b7adb213c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/HttpLLMInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/HttpLLMInterpreter.java @@ -9,8 +9,6 @@ import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; -import java.net.URI; -import java.net.URL; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -19,14 +17,16 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.net.URL; @Slf4j public class HttpLLMInterpreter implements LLMInterpreter { - public LLMResp query2sql(LLMReq llmReq, Long modelId) { + public LLMResp query2sql(LLMReq llmReq, String modelClusterKey) { long startTime = System.currentTimeMillis(); - log.info("requestLLM request, modelId:{},llmReq:{}", modelId, llmReq); + log.info("requestLLM request, modelId:{},llmReq:{}", modelClusterKey, llmReq); try { LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/LLMInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/LLMInterpreter.java index 701da8d54..1c36f51c1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/llm/LLMInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/llm/LLMInterpreter.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; public interface LLMInterpreter { - LLMResp query2sql(LLMReq llmReq, Long modelId); + LLMResp query2sql(LLMReq llmReq, String modelClusterKey); FunctionResp requestFunction(FunctionReq functionReq); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java index 8fae3e0f8..22f7f07d5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java @@ -9,12 +9,13 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; -import java.util.List; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.stream.Collectors; + /** * A mapper capable of converting the VALUE of entity dimension values into ID types. */ @@ -33,7 +34,8 @@ public class EntityMapper extends BaseMapper { if (entity == null || entity.getId() == null) { continue; } - List valueSchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch -> + List valueSchemaElements = schemaElementMatchList.stream() + .filter(schemaElementMatch -> SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) .collect(Collectors.toList()); for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/ModelClusterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/ModelClusterMapper.java new file mode 100644 index 000000000..f153d4ca3 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/ModelClusterMapper.java @@ -0,0 +1,52 @@ +package com.tencent.supersonic.chat.mapper; + +import com.tencent.supersonic.chat.api.component.SchemaMapper; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.utils.ModelClusterBuilder; +import com.tencent.supersonic.common.pojo.ModelCluster; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.knowledge.service.SchemaService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class ModelClusterMapper implements SchemaMapper { + + @Override + public void map(QueryContext queryContext) { + SchemaService schemaService = ContextUtils.getBean(SchemaService.class); + SemanticSchema semanticSchema = schemaService.getSemanticSchema(); + SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); + List modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema); + Map> modelClusterElementMatches = new HashMap<>(); + for (ModelCluster modelCluster : modelClusters) { + for (Long modelId : schemaMapInfo.getMatchedModels()) { + if (modelCluster.getModelIds().contains(modelId)) { + modelClusterElementMatches.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) + .addAll(schemaMapInfo.getMatchedElements(modelId)); + } + } + } + SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo(); + modelClusterMapInfo.setModelElementMatches(modelClusterElementMatches); + queryContext.setModelClusterMapInfo(modelClusterMapInfo); + } + + private List buildModelClusterMatched(SchemaMapInfo schemaMapInfo, + SemanticSchema semanticSchema) { + Set matchedModels = schemaMapInfo.getMatchedModels(); + List modelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema); + return modelClusters.stream().map(ModelCluster::getModelIds).peek(modelCluster -> { + modelCluster.removeIf(model -> !matchedModels.contains(model)); + }).filter(modelCluster -> modelCluster.size() > 0).map(ModelCluster::build).collect(Collectors.toList()); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java index 29077d8ce..ee747a3f6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; @@ -17,13 +16,14 @@ import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; /** * Query type parser, determine if the query is a metric query, an entity query, @@ -58,11 +58,10 @@ public class QueryTypeParser implements SemanticParser { //If all the fields in the SELECT statement are of tag type. List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL()); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = semanticService.getModelSchema(parseInfo.getModelId()); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); - if (CollectionUtils.isNotEmpty(selectFields) && Objects.nonNull(modelSchema) && CollectionUtils.isNotEmpty( - modelSchema.getTags())) { - Set tags = modelSchema.getTags().stream().map(schemaElement -> schemaElement.getName()) + if (CollectionUtils.isNotEmpty(selectFields)) { + Set tags = semanticSchema.getTags().stream().map(SchemaElement::getName) .collect(Collectors.toSet()); if (tags.containsAll(selectFields)) { return QueryType.TAG; @@ -72,10 +71,10 @@ public class QueryTypeParser implements SemanticParser { //2. metric queryType List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL()); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - List metrics = semanticSchema.getMetrics(parseInfo.getModelId()); + List metrics = semanticSchema.getMetrics(parseInfo.getModel().getModelIds()); if (CollectionUtils.isNotEmpty(metrics)) { - Set metricNameSet = metrics.stream().map(metric -> metric.getName()).collect(Collectors.toSet()); - boolean containMetric = selectFields.stream().anyMatch(selectField -> metricNameSet.contains(selectField)); + Set metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); + boolean containMetric = selectFields.stream().anyMatch(metricNameSet::contains); if (containMetric) { return QueryType.METRIC; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java index ce9749b79..fea2c1d28 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java @@ -5,10 +5,8 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.MetricInterpretTool; -import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; @@ -21,18 +19,19 @@ import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery; import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery; import com.tencent.supersonic.chat.service.AgentService; -import com.tencent.supersonic.chat.utils.ComponentFactory; +import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; @Slf4j public class MetricInterpretParser implements SemanticParser { @@ -81,9 +80,8 @@ public class MetricInterpretParser implements SemanticParser { } public Set getMetrics(List metricIds, Long modelId) { - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - ModelSchema modelSchema = semanticInterpreter.getModelSchema(modelId, true); - Set metrics = modelSchema.getMetrics(); + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + List metrics = semanticService.getSemanticSchema().getMetrics(); return metrics.stream().filter(schemaElement -> metricIds.contains(schemaElement.getId())) .collect(Collectors.toSet()); } @@ -112,16 +110,13 @@ public class MetricInterpretParser implements SemanticParser { private SemanticParseInfo buildSemanticParseInfo(Long modelId, QueryReq queryReq, Set metrics, List schemaElementMatches, String toolName) { - SchemaElement model = new SchemaElement(); - model.setModel(modelId); - model.setId(modelId); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setMetrics(metrics); SchemaElement dimension = new SchemaElement(); dimension.setBizName(TimeDimensionEnum.DAY.getName()); semanticParseInfo.setDimensions(Sets.newHashSet(dimension)); semanticParseInfo.setElementMatches(schemaElementMatches); - semanticParseInfo.setModel(model); + semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(modelId))); semanticParseInfo.setScore(queryReq.getQueryText().length()); DateConf dateConf = new DateConf(); dateConf.setDateMode(DateConf.DateMode.RECENT); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/HeuristicModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/HeuristicModelResolver.java index b60a27569..17259ab20 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/HeuristicModelResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/HeuristicModelResolver.java @@ -5,9 +5,11 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; +import com.tencent.supersonic.common.pojo.ModelCluster; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -17,31 +19,28 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; @Slf4j public class HeuristicModelResolver implements ModelResolver { - protected static Long selectModelBySchemaElementMatchScore(Map modelQueryModes, - SchemaMapInfo schemaMap) { + protected static String selectModelBySchemaElementMatchScore(Map modelQueryModes, + SchemaModelClusterMapInfo schemaMap) { //model count priority - Long modelIdByModelCount = getModelIdByMatchModelScore(schemaMap); + String modelIdByModelCount = getModelIdByMatchModelScore(schemaMap); if (Objects.nonNull(modelIdByModelCount)) { log.info("selectModel by model count:{}", modelIdByModelCount); return modelIdByModelCount; } - Map modelTypeMap = getModelTypeMap(schemaMap); + Map modelTypeMap = getModelTypeMap(schemaMap); if (modelTypeMap.size() == 1) { - Long modelSelect = modelTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey(); + String modelSelect = modelTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey(); if (modelQueryModes.containsKey(modelSelect)) { log.info("selectModel with only one Model [{}]", modelSelect); return modelSelect; } } else { - - Map.Entry maxModel = modelTypeMap.entrySet().stream() + Map.Entry maxModel = modelTypeMap.entrySet().stream() .filter(entry -> modelQueryModes.containsKey(entry.getKey())) .sorted((o1, o2) -> { int difference = o2.getValue().getCount() - o1.getValue().getCount(); @@ -56,16 +55,16 @@ public class HeuristicModelResolver implements ModelResolver { return maxModel.getKey(); } } - return 0L; + return null; } - private static Long getModelIdByMatchModelScore(SchemaMapInfo schemaMap) { - Map> modelElementMatches = schemaMap.getModelElementMatches(); + private static String getModelIdByMatchModelScore(SchemaModelClusterMapInfo schemaMap) { + Map> modelElementMatches = schemaMap.getModelElementMatches(); // calculate model match score, matched element gets 1.0 point, and inherit element gets 0.5 point - Map modelIdToModelScore = new HashMap<>(); + Map modelIdToModelScore = new HashMap<>(); if (Objects.nonNull(modelElementMatches)) { - for (Entry> modelElementMatch : modelElementMatches.entrySet()) { - Long modelId = modelElementMatch.getKey(); + for (Entry> modelElementMatch : modelElementMatches.entrySet()) { + String modelId = modelElementMatch.getKey(); List modelMatchesScore = modelElementMatch.getValue().stream() .filter(elementMatch -> elementMatch.getSimilarity() >= 1) .filter(elementMatch -> SchemaElementType.MODEL.equals(elementMatch.getElement().getType())) @@ -77,7 +76,7 @@ public class HeuristicModelResolver implements ModelResolver { modelIdToModelScore.put(modelId, score); } } - Entry maxModelScore = modelIdToModelScore.entrySet().stream() + Entry maxModelScore = modelIdToModelScore.entrySet().stream() .max(Comparator.comparingDouble(o -> o.getValue())).orElse(null); log.info("maxModelCount:{},modelIdToModelCount:{}", maxModelScore, modelIdToModelScore); if (Objects.nonNull(maxModelScore)) { @@ -87,64 +86,10 @@ public class HeuristicModelResolver implements ModelResolver { return null; } - /** - * to check can switch Model if context exit Model - * - * @return false will use context Model, true will use other Model , maybe include context Model - */ - protected static boolean isAllowSwitch(Map modelQueryModes, SchemaMapInfo schemaMap, - ChatContext chatCtx, QueryReq searchCtx, - Long modelId, Set restrictiveModels) { - if (!Objects.nonNull(modelId) || modelId <= 0) { - return true; - } - // except content Model, calculate the number of types for each Model, if numbers<=1 will not switch - Map modelTypeMap = getModelTypeMap(schemaMap); - log.info("isAllowSwitch ModelTypeMap [{}]", modelTypeMap); - long otherModelTypeNumBigOneCount = modelTypeMap.entrySet().stream() - .filter(entry -> modelQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(modelId)) - .filter(entry -> entry.getValue().getCount() > 1).count(); - if (otherModelTypeNumBigOneCount >= 1) { - return true; - } - // if query text only contain time , will not switch - if (!CollectionUtils.isEmpty(modelQueryModes.values())) { - for (SemanticQuery semanticQuery : modelQueryModes.values()) { - if (semanticQuery == null) { - continue; - } - SemanticParseInfo semanticParseInfo = semanticQuery.getParseInfo(); - if (semanticParseInfo == null) { - continue; - } - if (searchCtx.getQueryText() != null && semanticParseInfo.getDateInfo() != null) { - if (semanticParseInfo.getDateInfo().getDetectWord() != null) { - if (semanticParseInfo.getDateInfo().getDetectWord() - .equalsIgnoreCase(searchCtx.getQueryText())) { - log.info("timeParseResults is not null , can not switch context , timeParseResults:{},", - semanticParseInfo.getDateInfo()); - return false; - } - } - } - } - } - if (CollectionUtils.isNotEmpty(restrictiveModels) && !restrictiveModels.contains(modelId)) { - return true; - } - // if context Model not in schemaMap , will switch - if (schemaMap.getMatchedElements(modelId) == null || schemaMap.getMatchedElements(modelId).size() <= 0) { - log.info("modelId not in schemaMap "); - return true; - } - // other will not switch - return false; - } - - public static Map getModelTypeMap(SchemaMapInfo schemaMap) { - Map modelCount = new HashMap<>(); - for (Map.Entry> entry : schemaMap.getModelElementMatches().entrySet()) { + public static Map getModelTypeMap(SchemaModelClusterMapInfo schemaMap) { + Map modelCount = new HashMap<>(); + for (Map.Entry> entry : schemaMap.getModelElementMatches().entrySet()) { List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); if (schemaElementMatches != null && schemaElementMatches.size() > 0) { if (!modelCount.containsKey(entry.getKey())) { @@ -170,65 +115,34 @@ public class HeuristicModelResolver implements ModelResolver { } - public Long resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels) { + public String resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels) { + SchemaModelClusterMapInfo mapInfo = queryContext.getModelClusterMapInfo(); + Set matchedModelClusters = mapInfo.getElementMatchesByModelIds(restrictiveModels).keySet(); Long modelId = queryContext.getRequest().getModelId(); if (Objects.nonNull(modelId) && modelId > 0) { - if (CollectionUtils.isEmpty(restrictiveModels)) { - return modelId; - } - if (restrictiveModels.contains(modelId)) { - return modelId; - } else { - return null; + if (CollectionUtils.isEmpty(restrictiveModels) || restrictiveModels.contains(modelId)) { + return getModelClusterByModelId(modelId, matchedModelClusters); } + return null; } - SchemaMapInfo mapInfo = queryContext.getMapInfo(); - Set matchedModels = mapInfo.getMatchedModels(); - if (CollectionUtils.isNotEmpty(restrictiveModels)) { - matchedModels = matchedModels.stream() - .filter(restrictiveModels::contains) - .collect(Collectors.toSet()); - } - Map modelQueryModes = new HashMap<>(); - for (Long matchedModel : matchedModels) { + + Map modelQueryModes = new HashMap<>(); + for (String matchedModel : matchedModelClusters) { modelQueryModes.put(matchedModel, null); } if (modelQueryModes.size() == 1) { return modelQueryModes.keySet().stream().findFirst().get(); } - return resolve(modelQueryModes, queryContext, chatCtx, - queryContext.getMapInfo(), restrictiveModels); + return selectModelBySchemaElementMatchScore(modelQueryModes, mapInfo); } - public Long resolve(Map modelQueryModes, QueryContext queryContext, - ChatContext chatCtx, SchemaMapInfo schemaMap, Set restrictiveModels) { - Long selectModel = selectModel(modelQueryModes, queryContext.getRequest(), - chatCtx, schemaMap, restrictiveModels); - if (selectModel > 0) { - log.info("selectModel {} ", selectModel); - return selectModel; - } - // get the max SchemaElementType match score - return selectModelBySchemaElementMatchScore(modelQueryModes, schemaMap); - } - - public Long selectModel(Map modelQueryModes, QueryReq queryContext, - ChatContext chatCtx, - SchemaMapInfo schemaMap, Set restrictiveModels) { - // if QueryContext has modelId and in ModelQueryModes - if (modelQueryModes.containsKey(queryContext.getModelId())) { - log.info("selectModel from QueryContext [{}]", queryContext.getModelId()); - return queryContext.getModelId(); - } - // if ChatContext has modelId and in ModelQueryModes - if (chatCtx.getParseInfo().getModelId() > 0) { - Long modelId = chatCtx.getParseInfo().getModelId(); - if (!isAllowSwitch(modelQueryModes, schemaMap, chatCtx, queryContext, modelId, restrictiveModels)) { - log.info("selectModel from ChatContext [{}]", modelId); - return modelId; + private String getModelClusterByModelId(Long modelId, Set modelClusterKeySet) { + for (String modelClusterKey : modelClusterKeySet) { + if (ModelCluster.build(modelClusterKey).getModelIds().contains(modelId)) { + return modelClusterKey; } } - // default 0 - return 0L; + return null; } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMRequestService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMRequestService.java index 08f07c86f..03eaeec65 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMRequestService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMRequestService.java @@ -12,21 +12,28 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.config.LLMParserConfig; import com.tencent.supersonic.chat.config.OptimizationConfig; +import com.tencent.supersonic.chat.llm.LLMInterpreter; import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.chat.service.AgentService; -import com.tencent.supersonic.chat.llm.LLMInterpreter; import com.tencent.supersonic.chat.utils.ComponentFactory; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -35,12 +42,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Slf4j @Service @@ -72,18 +73,18 @@ public class LLMRequestService { return false; } - public Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) { + public ModelCluster getModelCluster(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) { Set distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2SQL); if (agentService.containsAllModel(distinctModelIds)) { distinctModelIds = new HashSet<>(); } ModelResolver modelResolver = ComponentFactory.getModelResolver(); - Long modelId = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds); - log.info("resolve modelId:{},llmParser Models:{}", modelId, distinctModelIds); - return modelId; + String modelCluster = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds); + log.info("resolve modelId:{},llmParser Models:{}", modelCluster, distinctModelIds); + return ModelCluster.build(modelCluster); } - public CommonAgentTool getParserTool(QueryReq request, Long modelId) { + public CommonAgentTool getParserTool(QueryReq request, Set modelIdSet) { List commonAgentTools = agentService.getParserTools(request.getAgentId(), AgentToolType.LLM_S2SQL); Optional llmParserTool = commonAgentTools.stream() @@ -92,31 +93,36 @@ public class LLMRequestService { if (agentService.containsAllModel(new HashSet<>(modelIds))) { return true; } - return modelIds.contains(modelId); + for (Long modelId : modelIdSet) { + if (modelIds.contains(modelId)) { + return true; + } + } + return false; }) .findFirst(); return llmParserTool.orElse(null); } - public LLMReq getLlmReq(QueryContext queryCtx, Long modelId, List linkingValues) { - SemanticSchema semanticSchema = schemaService.getSemanticSchema(); + public LLMReq getLlmReq(QueryContext queryCtx, SemanticSchema semanticSchema, + ModelCluster modelCluster, List linkingValues) { Map modelIdToName = semanticSchema.getModelIdToName(); String queryText = queryCtx.getRequest().getQueryText(); LLMReq llmReq = new LLMReq(); llmReq.setQueryText(queryText); - + Long firstModelId = modelCluster.getFirstModel(); LLMReq.FilterCondition filterCondition = new LLMReq.FilterCondition(); - filterCondition.setTableName(modelIdToName.get(modelId)); + filterCondition.setTableName(modelIdToName.get(firstModelId)); llmReq.setFilterCondition(filterCondition); LLMReq.LLMSchema llmSchema = new LLMReq.LLMSchema(); - llmSchema.setModelName(modelIdToName.get(modelId)); - llmSchema.setDomainName(modelIdToName.get(modelId)); + llmSchema.setModelName(modelIdToName.get(firstModelId)); + llmSchema.setDomainName(modelIdToName.get(firstModelId)); - List fieldNameList = getFieldNameList(queryCtx, modelId, llmParserConfig); + List fieldNameList = getFieldNameList(queryCtx, modelCluster, llmParserConfig); - String priorExts = getPriorExts(modelId, fieldNameList); + String priorExts = getPriorExts(modelCluster.getModelIds(), fieldNameList); llmReq.setPriorExts(priorExts); fieldNameList.add(TimeDimensionEnum.DAY.getChName()); @@ -129,7 +135,7 @@ public class LLMRequestService { } llmReq.setLinking(linking); - String currentDate = S2SQLDateHelper.getReferenceDate(modelId); + String currentDate = S2SQLDateHelper.getReferenceDate(firstModelId); if (StringUtils.isEmpty(currentDate)) { currentDate = DateUtils.getBeforeDate(0); } @@ -137,24 +143,25 @@ public class LLMRequestService { return llmReq; } - public LLMResp requestLLM(LLMReq llmReq, Long modelId) { - return llmInterpreter.query2sql(llmReq, modelId); + public LLMResp requestLLM(LLMReq llmReq, String modelClusterKey) { + return llmInterpreter.query2sql(llmReq, modelClusterKey); } - protected List getFieldNameList(QueryContext queryCtx, Long modelId, LLMParserConfig llmParserConfig) { + protected List getFieldNameList(QueryContext queryCtx, ModelCluster modelCluster, + LLMParserConfig llmParserConfig) { - Set results = getTopNFieldNames(modelId, llmParserConfig); + Set results = getTopNFieldNames(modelCluster, llmParserConfig); - Set fieldNameList = getMatchedFieldNames(queryCtx, modelId); + Set fieldNameList = getMatchedFieldNames(queryCtx, modelCluster); results.addAll(fieldNameList); return new ArrayList<>(results); } - private String getPriorExts(Long modelId, List fieldNameList) { + private String getPriorExts(Set modelIds, List fieldNameList) { StringBuilder extraInfoSb = new StringBuilder(); List modelSchemaResps = semanticInterpreter.fetchModelSchema( - Collections.singletonList(modelId), true); + new ArrayList<>(modelIds), true); if (!CollectionUtils.isEmpty(modelSchemaResps)) { ModelSchemaResp modelSchemaResp = modelSchemaResps.get(0); @@ -187,10 +194,11 @@ public class LLMRequestService { } - protected List getValueList(QueryContext queryCtx, Long modelId) { - Map itemIdToName = getItemIdToName(modelId); + protected List getValueList(QueryContext queryCtx, ModelCluster modelCluster) { + Map itemIdToName = getItemIdToName(modelCluster); - List matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId); + List matchedElements = queryCtx.getModelClusterMapInfo() + .getMatchedElements(modelCluster.getKey()); if (CollectionUtils.isEmpty(matchedElements)) { return new ArrayList<>(); } @@ -210,22 +218,22 @@ public class LLMRequestService { return new ArrayList<>(valueMatches); } - protected Map getItemIdToName(Long modelId) { + protected Map getItemIdToName(ModelCluster modelCluster) { SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - return semanticSchema.getDimensions(modelId).stream() + return semanticSchema.getDimensions(modelCluster.getModelIds()).stream() .collect(Collectors.toMap(SchemaElement::getId, SchemaElement::getName, (value1, value2) -> value2)); } - private Set getTopNFieldNames(Long modelId, LLMParserConfig llmParserConfig) { + private Set getTopNFieldNames(ModelCluster modelCluster, LLMParserConfig llmParserConfig) { SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - Set results = semanticSchema.getDimensions(modelId).stream() + Set results = semanticSchema.getDimensions(modelCluster.getModelIds()).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getDimensionTopN()) .map(entry -> entry.getName()) .collect(Collectors.toSet()); - Set metrics = semanticSchema.getMetrics(modelId).stream() + Set metrics = semanticSchema.getMetrics(modelCluster.getModelIds()).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getMetricTopN()) .map(entry -> entry.getName()) @@ -236,9 +244,10 @@ public class LLMRequestService { } - protected Set getMatchedFieldNames(QueryContext queryCtx, Long modelId) { - Map itemIdToName = getItemIdToName(modelId); - List matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId); + protected Set getMatchedFieldNames(QueryContext queryCtx, ModelCluster modelCluster) { + Map itemIdToName = getItemIdToName(modelCluster); + List matchedElements = queryCtx.getModelClusterMapInfo() + .getMatchedElements(modelCluster.getKey()); if (CollectionUtils.isEmpty(matchedElements)) { return new HashSet<>(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMResponseService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMResponseService.java index 6210989eb..601d42b59 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMResponseService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMResponseService.java @@ -2,24 +2,21 @@ package com.tencent.supersonic.chat.parser.llm.s2sql; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserEqualHelper; -import com.tencent.supersonic.knowledge.service.SchemaService; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.springframework.stereotype.Service; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + @Slf4j @Service public class LLMResponseService { @@ -30,9 +27,10 @@ public class LLMResponseService { } LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(S2SQLQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); - Long modelId = parseResult.getModelId(); + parseInfo.setModel(parseResult.getModelCluster()); CommonAgentTool commonAgentTool = parseResult.getCommonAgentTool(); - parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId)); + parseInfo.getElementMatches().addAll(queryCtx.getModelClusterMapInfo() + .getMatchedElements(parseInfo.getModelClusterKey())); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, parseResult); @@ -43,15 +41,7 @@ public class LLMResponseService { parseInfo.setScore(queryCtx.getRequest().getQueryText().length() * (1 + weight)); parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.getSqlInfo().setS2SQL(s2SQL); - - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - Map modelIdToName = semanticSchema.getModelIdToName(); - - SchemaElement model = new SchemaElement(); - model.setModel(modelId); - model.setId(modelId); - model.setName(modelIdToName.get(modelId)); - parseInfo.setModel(model); + parseInfo.setModel(parseResult.getModelCluster()); queryCtx.getCandidateQueries().add(semanticQuery); return parseInfo; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParser.java index dec947b72..db16f3c3c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParser.java @@ -4,16 +4,21 @@ import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; +import com.tencent.supersonic.chat.service.SemanticService; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.MapUtils; @Slf4j public class LLMS2SQLParser implements SemanticParser { @@ -22,36 +27,39 @@ public class LLMS2SQLParser implements SemanticParser { public void parse(QueryContext queryCtx, ChatContext chatCtx) { QueryReq request = queryCtx.getRequest(); LLMRequestService requestService = ContextUtils.getBean(LLMRequestService.class); + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); //1.determine whether to skip this parser. if (requestService.check(queryCtx)) { return; } try { //2.get modelId from queryCtx and chatCtx. - Long modelId = requestService.getModelId(queryCtx, chatCtx, request.getAgentId()); - if (Objects.isNull(modelId) || modelId <= 0) { + ModelCluster modelCluster = requestService.getModelCluster(queryCtx, chatCtx, request.getAgentId()); + if (StringUtils.isBlank(modelCluster.getKey())) { return; } //3.get agent tool and determine whether to skip this parser. - CommonAgentTool commonAgentTool = requestService.getParserTool(request, modelId); + CommonAgentTool commonAgentTool = requestService.getParserTool(request, modelCluster.getModelIds()); if (Objects.isNull(commonAgentTool)) { log.info("no tool in this agent, skip {}", LLMS2SQLParser.class); return; } //4.construct a request, call the API for the large model, and retrieve the results. - List linkingValues = requestService.getValueList(queryCtx, modelId); - LLMReq llmReq = requestService.getLlmReq(queryCtx, modelId, linkingValues); - LLMResp llmResp = requestService.requestLLM(llmReq, modelId); + List linkingValues = requestService.getValueList(queryCtx, modelCluster); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + LLMReq llmReq = requestService.getLlmReq(queryCtx, semanticSchema, modelCluster, linkingValues); + LLMResp llmResp = requestService.requestLLM(llmReq, modelCluster.getKey()); if (Objects.isNull(llmResp)) { return; } //5. deduplicate the SQL result list and build parserInfo + modelCluster.buildName(semanticSchema.getModelIdToName()); LLMResponseService responseService = ContextUtils.getBean(LLMResponseService.class); Map deduplicationSqlWeight = responseService.getDeduplicationSqlWeight(llmResp); ParseResult parseResult = ParseResult.builder() .request(request) - .modelId(modelId) + .modelCluster(modelCluster) .commonAgentTool(commonAgentTool) .llmReq(llmReq) .llmResp(llmResp) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelResolver.java index 901bab132..58c995fc9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelResolver.java @@ -7,6 +7,6 @@ import java.util.Set; public interface ModelResolver { - Long resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels); + String resolve(QueryContext queryContext, ChatContext chatCtx, Set restrictiveModels); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ParseResult.java index 65914f82d..0f50db29e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ParseResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ParseResult.java @@ -5,19 +5,21 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp; -import java.util.List; +import com.tencent.supersonic.common.pojo.ModelCluster; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @Builder @AllArgsConstructor @NoArgsConstructor public class ParseResult { - private Long modelId; + private ModelCluster modelCluster; private LLMReq llmReq; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java index 2e5c99673..587b4facb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java @@ -7,11 +7,10 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginParseResult; @@ -19,8 +18,10 @@ import com.tencent.supersonic.chat.plugin.PluginRecallResult; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import org.springframework.util.CollectionUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -58,8 +59,10 @@ public abstract class PluginParser implements SemanticParser { } for (Long modelId : modelIds) { PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); - SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, queryContext.getRequest(), - queryContext.getMapInfo().getMatchedElements(modelId), pluginRecallResult.getDistance()); + SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, + queryContext.getRequest(), + queryContext.getModelClusterMapInfo().getMatchedElements(modelId), + pluginRecallResult.getDistance()); semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); semanticParseInfo.setScore(pluginRecallResult.getScore()); pluginQuery.setParseInfo(semanticParseInfo); @@ -79,12 +82,9 @@ public abstract class PluginParser implements SemanticParser { if (schemaElementMatches == null) { schemaElementMatches = Lists.newArrayList(); } - SchemaElement model = new SchemaElement(); - model.setModel(modelId); - model.setId(modelId); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setElementMatches(schemaElementMatches); - semanticParseInfo.setModel(model); + semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(modelId))); Map properties = new HashMap<>(); PluginParseResult pluginParseResult = new PluginParseResult(); pluginParseResult.setPlugin(plugin); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java index 9b20eb9c3..2199bd902 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AgentCheckParser.java @@ -13,6 +13,8 @@ import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.common.util.ContextUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; + +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; @@ -47,7 +49,11 @@ public class AgentCheckParser implements SemanticParser { if (CollectionUtils.isEmpty(tool.getModelIds())) { return true; } - if (tool.isContainsAllModel() || tool.getModelIds().contains(query.getParseInfo().getModelId())) { + if (tool.isContainsAllModel()) { + return false; + } + if (new HashSet<>(tool.getModelIds()) + .containsAll(query.getParseInfo().getModel().getModelIds())) { return false; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java index 07873c261..27c493a96 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java @@ -1,32 +1,41 @@ package com.tencent.supersonic.chat.parser.rule; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.MODEL; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; - import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.query.rule.metric.MetricTagQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricTagQuery; +import com.tencent.supersonic.chat.service.SemanticService; +import com.tencent.supersonic.chat.utils.ModelClusterBuilder; +import com.tencent.supersonic.common.pojo.ModelCluster; +import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.MODEL; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.TAG; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; /** * ContextInheritParser tries to inherit certain schema elements from context @@ -41,20 +50,22 @@ public class ContextInheritParser implements SemanticParser { new AbstractMap.SimpleEntry<>(DIMENSION, Arrays.asList(DIMENSION, VALUE)), new AbstractMap.SimpleEntry<>(VALUE, Arrays.asList(VALUE, DIMENSION)), new AbstractMap.SimpleEntry<>(ENTITY, Arrays.asList(ENTITY)), + new AbstractMap.SimpleEntry<>(TAG, Arrays.asList(TAG)), new AbstractMap.SimpleEntry<>(MODEL, Arrays.asList(MODEL)), new AbstractMap.SimpleEntry<>(ID, Arrays.asList(ID)) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @Override public void parse(QueryContext queryContext, ChatContext chatContext) { - if (!shouldInherit(queryContext, chatContext)) { + if (!shouldInherit(queryContext)) { return; } - - Long modelId = chatContext.getParseInfo().getModelId(); - List elementMatches = queryContext.getMapInfo() - .getMatchedElements(modelId); - + ModelCluster modelCluster = getMatchedModelCluster(queryContext, chatContext); + if (modelCluster == null) { + return; + } + List elementMatches = queryContext.getModelClusterMapInfo() + .getMatchedElements(modelCluster.getKey()); List matchesToInherit = new ArrayList<>(); for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) { SchemaElementType matchType = match.getElement().getType(); @@ -69,18 +80,18 @@ public class ContextInheritParser implements SemanticParser { List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { - query.fillParseInfo(modelId, queryContext, chatContext); - if (existSameQuery(query.getParseInfo().getModelId(), query.getQueryMode(), queryContext)) { + query.fillParseInfo(chatContext); + if (existSameQuery(query.getParseInfo().getModelClusterKey(), query.getQueryMode(), queryContext)) { continue; } queryContext.getCandidateQueries().add(query); } } - private boolean existSameQuery(Long modelId, String queryMode, QueryContext queryContext) { + private boolean existSameQuery(String modelClusterKey, String queryMode, QueryContext queryContext) { for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) { if (semanticQuery.getQueryMode().equals(queryMode) - && semanticQuery.getParseInfo().getModelId().equals(modelId)) { + && semanticQuery.getParseInfo().getModelClusterKey().equals(modelClusterKey)) { return true; } } @@ -101,23 +112,34 @@ public class ContextInheritParser implements SemanticParser { }); } - protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) { - Long contextModelId = chatContext.getParseInfo().getModelId(); - // if map info doesn't contain the same Model of the context, - // no inheritance could be done - if (queryContext.getMapInfo().getMatchedElements(contextModelId) == null) { - return false; - } - + protected boolean shouldInherit(QueryContext queryContext) { // if candidates only have MetricModel mode, count in context List metricModelQueries = queryContext.getCandidateQueries().stream() .filter(query -> query instanceof MetricModelQuery).collect( Collectors.toList()); - if (metricModelQueries.size() == queryContext.getCandidateQueries().size()) { - return true; - } else { - return queryContext.getCandidateQueries().size() == 0; + return metricModelQueries.size() == queryContext.getCandidateQueries().size(); + } + + protected ModelCluster getMatchedModelCluster(QueryContext queryContext, ChatContext chatContext) { + String contextModelClusterKey = chatContext.getParseInfo().getModelClusterKey(); + if (StringUtils.isBlank(contextModelClusterKey)) { + return null; } + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + List allModelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema); + Set queryModelClusters = queryContext.getModelClusterMapInfo().getMatchedModelClusters(); + ModelCluster contextModelCluster = ModelCluster.build(contextModelClusterKey); + for (String cluster : queryModelClusters) { + ModelCluster queryModelCluster = ModelCluster.build(cluster); + for (ModelCluster modelCluster : allModelClusters) { + if (modelCluster.getModelIds().containsAll(contextModelCluster.getModelIds()) + && modelCluster.getModelIds().containsAll(queryModelCluster.getModelIds())) { + return queryModelCluster; + } + } + } + return null; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java index 4ff9ec839..b8435776b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java @@ -1,16 +1,15 @@ package com.tencent.supersonic.chat.parser.rule; import com.tencent.supersonic.chat.api.component.SemanticParser; -import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; +import lombok.extern.slf4j.Slf4j; import java.util.List; -import lombok.extern.slf4j.Slf4j; - /** * QueryModeParser resolves a specific query mode according to co-appearance * of certain schema element types. @@ -20,13 +19,13 @@ public class QueryModeParser implements SemanticParser { @Override public void parse(QueryContext queryContext, ChatContext chatContext) { - SchemaMapInfo mapInfo = queryContext.getMapInfo(); + SchemaModelClusterMapInfo modelClusterMapInfo = queryContext.getModelClusterMapInfo(); // iterate all schemaElementMatches to resolve query mode - for (Long modelId : mapInfo.getMatchedModels()) { - List elementMatches = mapInfo.getMatchedElements(modelId); + for (String modelClusterKey : modelClusterMapInfo.getMatchedModelClusters()) { + List elementMatches = modelClusterMapInfo.getMatchedElements(modelClusterKey); List queries = RuleSemanticQuery.resolve(elementMatches, queryContext); for (RuleSemanticQuery query : queries) { - query.fillParseInfo(modelId, queryContext, chatContext); + query.fillParseInfo(chatContext); queryContext.getCandidateQueries().add(query); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java index b89d2fbcf..299cdf34c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java @@ -3,24 +3,23 @@ package com.tencent.supersonic.chat.postprocessor; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.chat.api.component.SemanticQuery; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.knowledge.service.SchemaService; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; + import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -34,17 +33,15 @@ public class MetricCheckPostProcessor implements PostProcessor { @Override public void process(QueryContext queryContext) { List semanticQueries = queryContext.getCandidateQueries(); - Map modelSchemaMap = new HashMap<>(); + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticSchema semanticSchema = semanticService.getSemanticSchema(); for (SemanticQuery semanticQuery : semanticQueries) { SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); if (!QueryType.METRIC.equals(parseInfo.getQueryType())) { continue; } - SchemaService schemaService = ContextUtils.getBean(SchemaService.class); - ModelSchema modelSchema = schemaService.getModelSchema(parseInfo.getModelId()); - String processedSql = processCorrectSql(parseInfo.getSqlInfo().getCorrectS2SQL(), modelSchema); - parseInfo.getSqlInfo().setCorrectS2SQL(processedSql); - modelSchemaMap.put(modelSchema.getModel().getModel(), modelSchema); + String correctSqlProcessed = processCorrectSql(parseInfo, semanticSchema); + parseInfo.getSqlInfo().setCorrectS2SQL(correctSqlProcessed); } semanticQueries.removeIf(semanticQuery -> { if (!QueryType.METRIC.equals(semanticQuery.getParseInfo().getQueryType())) { @@ -54,14 +51,14 @@ public class MetricCheckPostProcessor implements PostProcessor { if (StringUtils.isBlank(correctSql)) { return false; } - return !checkHasMetric(correctSql, modelSchemaMap.get(semanticQuery.getParseInfo().getModelId())); + return !checkHasMetric(correctSql, semanticSchema); }); } - public String processCorrectSql(String correctSql, ModelSchema modelSchema) { + public String processCorrectSql(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { + String correctSql = parseInfo.getSqlInfo().getCorrectS2SQL(); List groupByFields = SqlParserSelectHelper.getGroupByFields(correctSql); - List metricFields = SqlParserSelectHelper.getAggregateFields(correctSql) - .stream().filter(metricField -> !metricField.equals("*")).collect(Collectors.toList()); + List metricFields = SqlParserSelectHelper.getAggregateFields(correctSql); List whereFields = SqlParserSelectHelper.getWhereFields(correctSql); List dimensionFields = getDimensionFields(groupByFields, whereFields); if (CollectionUtils.isEmpty(metricFields) || StringUtils.isBlank(correctSql)) { @@ -71,35 +68,33 @@ public class MetricCheckPostProcessor implements PostProcessor { Set groupByToRemove = Sets.newHashSet(); Set whereFieldsToRemove = Sets.newHashSet(); for (String metricName : metricFields) { - SchemaElement metricElement = modelSchema.getElement(SchemaElementType.METRIC, metricName); + SchemaElement metricElement = semanticSchema.getElementByName(SchemaElementType.METRIC, metricName); if (metricElement == null) { metricToRemove.add(metricName); } - if (!checkNecessaryDimension(metricElement, modelSchema, dimensionFields)) { + if (!checkNecessaryDimension(metricElement, semanticSchema, dimensionFields)) { metricToRemove.add(metricName); } } for (String dimensionName : whereFields) { - if (TimeDimensionEnum.getNameList().contains(dimensionName) - || TimeDimensionEnum.getChNameList().contains(dimensionName)) { + if (TimeDimensionEnum.getNameList().contains(dimensionName)) { continue; } - if (!checkInModelSchema(dimensionName, SchemaElementType.DIMENSION, modelSchema)) { + if (!checkInModelSchema(dimensionName, SchemaElementType.DIMENSION, semanticSchema)) { whereFieldsToRemove.add(dimensionName); } - if (!checkDrillDownDimension(dimensionName, metricFields, modelSchema)) { + if (!checkDrillDownDimension(dimensionName, metricFields, semanticSchema)) { whereFieldsToRemove.add(dimensionName); } } for (String dimensionName : groupByFields) { - if (TimeDimensionEnum.getNameList().contains(dimensionName) - || TimeDimensionEnum.getChNameList().contains(dimensionName)) { + if (TimeDimensionEnum.getNameList().contains(dimensionName)) { continue; } - if (!checkInModelSchema(dimensionName, SchemaElementType.DIMENSION, modelSchema)) { + if (!checkInModelSchema(dimensionName, SchemaElementType.DIMENSION, semanticSchema)) { groupByToRemove.add(dimensionName); } - if (!checkDrillDownDimension(dimensionName, metricFields, modelSchema)) { + if (!checkDrillDownDimension(dimensionName, metricFields, semanticSchema)) { groupByToRemove.add(dimensionName); } } @@ -111,9 +106,9 @@ public class MetricCheckPostProcessor implements PostProcessor { * To check whether the dimension bound to the metric exists, * eg: metric like UV is calculated in a certain dimension, it cannot be used on other dimensions. */ - private boolean checkNecessaryDimension(SchemaElement metric, ModelSchema modelSchema, + private boolean checkNecessaryDimension(SchemaElement metric, SemanticSchema semanticSchema, List dimensionFields) { - List necessaryDimensions = getNecessaryDimensionNames(metric, modelSchema); + List necessaryDimensions = getNecessaryDimensionNames(metric, semanticSchema); if (CollectionUtils.isEmpty(necessaryDimensions)) { return true; } @@ -130,8 +125,8 @@ public class MetricCheckPostProcessor implements PostProcessor { * eg: some descriptive dimensions are not suitable as drill-down dimensions */ private boolean checkDrillDownDimension(String dimensionName, List metrics, - ModelSchema modelSchema) { - List metricElements = modelSchema.getMetrics().stream() + SemanticSchema semanticSchema) { + List metricElements = semanticSchema.getMetrics().stream() .filter(schemaElement -> metrics.contains(schemaElement.getName())) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(metricElements)) { @@ -142,7 +137,7 @@ public class MetricCheckPostProcessor implements PostProcessor { .map(schemaElement -> schemaElement.getRelateSchemaElements().stream() .map(RelateSchemaElement::getDimensionId).collect(Collectors.toList())) .flatMap(Collection::stream) - .map(id -> convertDimensionIdToName(id, modelSchema)) + .map(id -> convertDimensionIdToName(id, semanticSchema)) .filter(Objects::nonNull) .collect(Collectors.toList()); //if no metric has drill down dimension, return true @@ -153,9 +148,9 @@ public class MetricCheckPostProcessor implements PostProcessor { return relateDimensions.contains(dimensionName); } - private List getNecessaryDimensionNames(SchemaElement metric, ModelSchema modelSchema) { + private List getNecessaryDimensionNames(SchemaElement metric, SemanticSchema semanticSchema) { List necessaryDimensionIds = getNecessaryDimensions(metric); - return necessaryDimensionIds.stream().map(id -> convertDimensionIdToName(id, modelSchema)) + return necessaryDimensionIds.stream().map(id -> convertDimensionIdToName(id, semanticSchema)) .filter(Objects::nonNull).collect(Collectors.toList()); } @@ -183,23 +178,23 @@ public class MetricCheckPostProcessor implements PostProcessor { return dimensionFields; } - private String convertDimensionIdToName(Long id, ModelSchema modelSchema) { - SchemaElement schemaElement = modelSchema.getElement(SchemaElementType.DIMENSION, id); + private String convertDimensionIdToName(Long id, SemanticSchema semanticSchema) { + SchemaElement schemaElement = semanticSchema.getElement(SchemaElementType.DIMENSION, id); if (schemaElement == null) { return null; } return schemaElement.getName(); } - private boolean checkInModelSchema(String name, SchemaElementType type, ModelSchema modelSchema) { - SchemaElement schemaElement = modelSchema.getElement(type, name); + private boolean checkInModelSchema(String name, SchemaElementType type, SemanticSchema semanticSchema) { + SchemaElement schemaElement = semanticSchema.getElementByName(type, name); return schemaElement != null; } - private boolean checkHasMetric(String correctSql, ModelSchema modelSchema) { + private boolean checkHasMetric(String correctSql, SemanticSchema semanticSchema) { List selectFields = SqlParserSelectHelper.getSelectFields(correctSql); List aggFields = SqlParserSelectHelper.getAggregateFields(correctSql); - List collect = modelSchema.getMetrics().stream() + List collect = semanticSchema.getMetrics().stream() .map(SchemaElement::getName).collect(Collectors.toList()); for (String field : selectFields) { if (collect.contains(field)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java index 566a91d6d..a99a414d0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java @@ -1,17 +1,40 @@ package com.tencent.supersonic.chat.postprocessor; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.service.ParseInfoService; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; +import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryType; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.knowledge.service.SchemaService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** * update parse info from correct sql */ +@Slf4j public class ParseInfoUpdateProcessor implements PostProcessor { @Override @@ -20,10 +43,174 @@ public class ParseInfoUpdateProcessor implements PostProcessor { if (CollectionUtils.isEmpty(candidateQueries)) { return; } - ParseInfoService parseInfoService = ContextUtils.getBean(ParseInfoService.class); List candidateParses = candidateQueries.stream() .map(SemanticQuery::getParseInfo).collect(Collectors.toList()); - candidateParses.forEach(parseInfoService::updateParseInfo); + candidateParses.forEach(this::updateParseInfo); + } + public void updateParseInfo(SemanticParseInfo parseInfo) { + SqlInfo sqlInfo = parseInfo.getSqlInfo(); + String correctS2SQL = sqlInfo.getCorrectS2SQL(); + if (StringUtils.isBlank(correctS2SQL)) { + return; + } + // if S2SQL equals correctS2SQL, than not update the parseInfo. + if (correctS2SQL.equals(sqlInfo.getS2SQL())) { + return; + } + + List expressions = SqlParserSelectHelper.getFilterExpression(correctS2SQL); + //set dataInfo + try { + if (!org.apache.commons.collections.CollectionUtils.isEmpty(expressions)) { + DateConf dateInfo = getDateInfo(expressions); + if (dateInfo != null && parseInfo.getDateInfo() == null) { + parseInfo.setDateInfo(dateInfo); + } + } + } catch (Exception e) { + log.error("set dateInfo error :", e); + } + + //set filter + try { + Map fieldNameToElement = getNameToElement(parseInfo.getModel().getModelIds()); + List result = getDimensionFilter(fieldNameToElement, expressions); + parseInfo.getDimensionFilters().addAll(result); + } catch (Exception e) { + log.error("set dimensionFilter error :", e); + } + + SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); + + if (Objects.isNull(semanticSchema)) { + return; + } + List allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); + Set metrics = getElements(parseInfo.getModel().getModelIds(), + allFields, semanticSchema.getMetrics()); + parseInfo.setMetrics(metrics); + + if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) { + parseInfo.setQueryType(QueryType.METRIC); + List groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); + List groupByDimensions = getFieldsExceptDate(groupByFields); + parseInfo.setDimensions( + getElements(parseInfo.getModel().getModelIds(), groupByDimensions, semanticSchema.getDimensions())); + } else { + parseInfo.setQueryType(QueryType.TAG); + List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); + List selectDimensions = getFieldsExceptDate(selectFields); + parseInfo.setDimensions( + getElements(parseInfo.getModel().getModelIds(), selectDimensions, semanticSchema.getDimensions())); + } + } + + + private Set getElements(Set modelIds, List allFields, List elements) { + return elements.stream() + .filter(schemaElement -> modelIds.contains(schemaElement.getModel()) + && allFields.contains(schemaElement.getName()) + ).collect(Collectors.toSet()); + } + + private List getFieldsExceptDate(List allFields) { + if (org.springframework.util.CollectionUtils.isEmpty(allFields)) { + return new ArrayList<>(); + } + return allFields.stream() + .filter(entry -> !TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(entry)) + .collect(Collectors.toList()); + } + + + private List getDimensionFilter(Map fieldNameToElement, + List fieldExpressions) { + List result = Lists.newArrayList(); + for (FieldExpression expression : fieldExpressions) { + QueryFilter dimensionFilter = new QueryFilter(); + dimensionFilter.setValue(expression.getFieldValue()); + SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName()); + if (Objects.isNull(schemaElement)) { + continue; + } + dimensionFilter.setName(schemaElement.getName()); + dimensionFilter.setBizName(schemaElement.getBizName()); + dimensionFilter.setElementID(schemaElement.getId()); + + FilterOperatorEnum operatorEnum = FilterOperatorEnum.getSqlOperator(expression.getOperator()); + dimensionFilter.setOperator(operatorEnum); + dimensionFilter.setFunction(expression.getFunction()); + result.add(dimensionFilter); + } + return result; + } + + private DateConf getDateInfo(List fieldExpressions) { + List dateExpressions = fieldExpressions.stream() + .filter(expression -> TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(expression.getFieldName())) + .collect(Collectors.toList()); + if (org.apache.commons.collections.CollectionUtils.isEmpty(dateExpressions)) { + return null; + } + DateConf dateInfo = new DateConf(); + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + FieldExpression firstExpression = dateExpressions.get(0); + + FilterOperatorEnum firstOperator = FilterOperatorEnum.getSqlOperator(firstExpression.getOperator()); + if (FilterOperatorEnum.EQUALS.equals(firstOperator) && Objects.nonNull(firstExpression.getFieldValue())) { + dateInfo.setStartDate(firstExpression.getFieldValue().toString()); + dateInfo.setEndDate(firstExpression.getFieldValue().toString()); + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + return dateInfo; + } + if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN, + FilterOperatorEnum.GREATER_THAN_EQUALS)) { + dateInfo.setStartDate(firstExpression.getFieldValue().toString()); + if (hasSecondDate(dateExpressions)) { + dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString()); + } + } + if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN, + FilterOperatorEnum.MINOR_THAN_EQUALS)) { + dateInfo.setEndDate(firstExpression.getFieldValue().toString()); + if (hasSecondDate(dateExpressions)) { + dateInfo.setStartDate(dateExpressions.get(1).getFieldValue().toString()); + } + } + return dateInfo; + } + + private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator, + FilterOperatorEnum... operatorEnums) { + return (Arrays.asList(operatorEnums).contains(firstOperator) && Objects.nonNull(expression.getFieldValue())); + } + + private boolean hasSecondDate(List dateExpressions) { + return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); + } + + protected Map getNameToElement(Set modelIds) { + SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); + List dimensions = semanticSchema.getDimensions(modelIds); + List metrics = semanticSchema.getMetrics(modelIds); + + List allElements = Lists.newArrayList(); + allElements.addAll(dimensions); + allElements.addAll(metrics); + //support alias + return allElements.stream() + .flatMap(schemaElement -> { + Set> result = new HashSet<>(); + result.add(Pair.of(schemaElement.getName(), schemaElement)); + List aliasList = schemaElement.getAlias(); + if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) { + for (String alias : aliasList) { + result.add(Pair.of(alias, schemaElement)); + } + } + return result.stream(); + }) + .collect(Collectors.toMap(pair -> pair.getLeft(), pair -> pair.getRight(), (value1, value2) -> value2)); } } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/BaseSemanticQuery.java index eaebb11a4..e384dd62c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/BaseSemanticQuery.java @@ -20,15 +20,16 @@ import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; @Slf4j @ToString @@ -49,7 +50,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryTypeEnum.SQL) .queryReq(QueryReqBuilder.buildS2SQLReq( - sqlInfo.getCorrectS2SQL(), parseInfo.getModelId() + sqlInfo.getCorrectS2SQL(), parseInfo.getModel().getModelIds() )) .build(); } else { @@ -86,7 +87,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { protected void convertBizNameToName(QueryStructReq queryStructReq) { SchemaService schemaService = ContextUtils.getBean(SchemaService.class); Map bizNameToName = schemaService.getSemanticSchema() - .getBizNameToName(queryStructReq.getModelId()); + .getBizNameToName(queryStructReq.getModelIdSet()); bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap()); List orders = queryStructReq.getOrders(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryRanker.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryRanker.java deleted file mode 100644 index f7d51546d..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryRanker.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.tencent.supersonic.chat.query; - -import com.tencent.supersonic.chat.api.component.SemanticQuery; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -@Component -public class QueryRanker { - - @Value("${candidate.top.size:5}") - private int candidateTopSize; - - public List rank(List candidateQueries) { - log.debug("pick before [{}]", candidateQueries); - if (CollectionUtils.isEmpty(candidateQueries)) { - return candidateQueries; - } - List selectedQueries = new ArrayList<>(); - if (candidateQueries.size() == 1) { - selectedQueries.addAll(candidateQueries); - } else { - selectedQueries = getTopCandidateQuery(candidateQueries); - } - generateParseInfoId(selectedQueries); - log.debug("pick after [{}]", selectedQueries); - return selectedQueries; - } - - public List getTopCandidateQuery(List semanticQueries) { - return semanticQueries.stream() - .filter(query -> !checkFullyInherited(query)) - .sorted((o1, o2) -> { - if (o1.getParseInfo().getScore() < o2.getParseInfo().getScore()) { - return 1; - } else if (o1.getParseInfo().getScore() > o2.getParseInfo().getScore()) { - return -1; - } - return 0; - }).limit(candidateTopSize) - .collect(Collectors.toList()); - } - - private void generateParseInfoId(List semanticQueries) { - for (int i = 0; i < semanticQueries.size(); i++) { - SemanticQuery query = semanticQueries.get(i); - query.getParseInfo().setId(i + 1); - } - } - - private boolean checkFullyInherited(SemanticQuery query) { - SemanticParseInfo parseInfo = query.getParseInfo(); - if (!(query instanceof RuleSemanticQuery)) { - return false; - } - - for (SchemaElementMatch match : parseInfo.getElementMatches()) { - if (!match.isInherited()) { - return false; - } - } - return parseInfo.getDateInfo() == null || parseInfo.getDateInfo().isInherited(); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/S2SQLQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/S2SQLQuery.java index 7d52e370a..87a2de53b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/S2SQLQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/S2SQLQuery.java @@ -12,12 +12,13 @@ import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; @Slf4j @Component @@ -40,7 +41,7 @@ public class S2SQLQuery extends LLMSemanticQuery { long startTime = System.currentTimeMillis(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModelId()); + QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2SQL(queryS2SQLReq, user); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java index f706e30bf..6cf8143e4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.query.plugin.webpage; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; @@ -17,17 +16,15 @@ import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.query.plugin.WebBaseResult; -import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.HashMap; @Slf4j @Component @@ -53,9 +50,6 @@ public class WebPageQuery extends PluginSemanticQuery { PluginParseResult.class); WebPageResponse webPageResponse = buildResponse(pluginParseResult); queryResult.setResponse(webPageResponse); - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = semanticService.getModelSchema(parseInfo.getModelId()); - parseInfo.setModel(modelSchema.getModel()); queryResult.setQueryState(QueryState.SUCCESS); return queryResult; } @@ -79,7 +73,8 @@ public class WebPageQuery extends PluginSemanticQuery { List paramOptions = Lists.newArrayList(); if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) { for (ParamOption paramOption : webPage.getParamOptions()) { - if (paramOption.getModelId() != null && !paramOption.getModelId().equals(parseInfo.getModelId())) { + if (paramOption.getModelId() != null + && !parseInfo.getModel().getModelIds().contains(paramOption.getModelId())) { continue; } paramOptions.add(paramOption); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index b24035d64..a62a6850b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -4,12 +4,12 @@ package com.tencent.supersonic.chat.query.rule; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; @@ -19,21 +19,25 @@ import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import java.util.Set; +import java.util.stream.Collectors; @Slf4j @ToString @@ -56,13 +60,13 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { initS2SqlByStruct(); } - public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { + public void fillParseInfo(ChatContext chatContext) { parseInfo.setQueryMode(getQueryMode()); SemanticService schemaService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = schemaService.getModelSchema(modelId); + SemanticSchema semanticSchema = schemaService.getSemanticSchema(); - fillSchemaElement(parseInfo, modelSchema); + fillSchemaElement(parseInfo, semanticSchema); fillScore(parseInfo); fillDateConf(parseInfo, chatContext.getParseInfo()); } @@ -101,9 +105,12 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { parseInfo.setScore(parseInfo.getScore() + totalScore); } - private void fillSchemaElement(SemanticParseInfo parseInfo, ModelSchema modelSchema) { - parseInfo.setModel(modelSchema.getModel()); - + private void fillSchemaElement(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { + Set modelIds = parseInfo.getElementMatches().stream().map(SchemaElementMatch::getElement) + .map(SchemaElement::getModel).collect(Collectors.toSet()); + ModelCluster modelCluster = ModelCluster.build(modelIds); + modelCluster.buildName(semanticSchema.getModelIdToName()); + parseInfo.setModel(modelCluster); Map> dim2Values = new HashMap<>(); Map> id2Values = new HashMap<>(); @@ -112,7 +119,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { element.setOrder(1 - schemaMatch.getSimilarity()); switch (element.getType()) { case ID: - SchemaElement entityElement = modelSchema.getElement(SchemaElementType.ENTITY, element.getId()); + SchemaElement entityElement = semanticSchema.getElement(SchemaElementType.ENTITY, element.getId()); if (entityElement != null) { if (id2Values.containsKey(element.getId())) { id2Values.get(element.getId()).add(schemaMatch); @@ -122,7 +129,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { } break; case VALUE: - SchemaElement dimElement = modelSchema.getElement(SchemaElementType.DIMENSION, element.getId()); + SchemaElement dimElement = semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId()); if (dimElement != null) { if (dim2Values.containsKey(element.getId())) { dim2Values.get(element.getId()).add(schemaMatch); @@ -146,20 +153,20 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { if (!id2Values.isEmpty()) { for (Map.Entry> entry : id2Values.entrySet()) { - addFilters(parseInfo, modelSchema, entry, SchemaElementType.ENTITY); + addFilters(parseInfo, semanticSchema, entry, SchemaElementType.ENTITY); } } if (!dim2Values.isEmpty()) { for (Map.Entry> entry : dim2Values.entrySet()) { - addFilters(parseInfo, modelSchema, entry, SchemaElementType.DIMENSION); + addFilters(parseInfo, semanticSchema, entry, SchemaElementType.DIMENSION); } } } - private void addFilters(SemanticParseInfo parseInfo, ModelSchema modelSchema, - Entry> entry, SchemaElementType dimension1) { - SchemaElement dimension = modelSchema.getElement(dimension1, entry.getKey()); + private void addFilters(SemanticParseInfo parseInfo, SemanticSchema semanticSchema, + Entry> entry, SchemaElementType elementType) { + SchemaElement dimension = semanticSchema.getElement(elementType, entry.getKey()); if (entry.getValue().size() == 1) { SchemaElementMatch schemaMatch = entry.getValue().get(0); @@ -170,7 +177,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); dimensionFilter.setElementID(schemaMatch.getElement().getId()); parseInfo.getDimensionFilters().add(dimensionFilter); - parseInfo.setEntity(modelSchema.getEntity()); + parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey())); } else { QueryFilter dimensionFilter = new QueryFilter(); List vals = new ArrayList<>(); @@ -189,7 +196,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult execute(User user) { String queryMode = parseInfo.getQueryMode(); - if (parseInfo.getModelId() < 0 || StringUtils.isEmpty(queryMode) + if (StringUtils.isBlank(parseInfo.getModelClusterKey()) || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); @@ -230,7 +237,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { public QueryResult multiStructExecute(User user) { String queryMode = parseInfo.getQueryMode(); - if (parseInfo.getModelId() < 0 || StringUtils.isEmpty(queryMode) + if (StringUtils.isBlank(parseInfo.getModelClusterKey()) || StringUtils.isEmpty(queryMode) || !QueryManager.containsRuleQuery(queryMode)) { // reach here some error may happen log.error("not find QueryMode"); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java index 61f66bf3f..11cfac9b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.chat.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; @@ -20,12 +17,17 @@ import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { @@ -82,8 +84,8 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { @Override - public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { - super.fillParseInfo(modelId, queryContext, chatContext); + public void fillParseInfo(ChatContext chatContext) { + super.fillParseInfo(chatContext); parseInfo.setLimit(METRIC_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTagQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTagQuery.java index e6575cb1e..0f0c0457f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTagQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTagQuery.java @@ -25,7 +25,7 @@ import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNum @Component public class MetricTagQuery extends MetricSemanticQuery { - public static final String QUERY_MODE = "TAG_ENTITY"; + public static final String QUERY_MODE = "METRIC_ENTITY"; public MetricTagQuery() { super(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java index 8c515231d..93c07a464 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java @@ -1,12 +1,5 @@ package com.tencent.supersonic.chat.query.rule.metric; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; - import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; @@ -20,6 +13,13 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; +import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; + @Component public class MetricTopNQuery extends MetricSemanticQuery { @@ -50,8 +50,8 @@ public class MetricTopNQuery extends MetricSemanticQuery { } @Override - public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { - super.fillParseInfo(modelId, queryContext, chatContext); + public void fillParseInfo(ChatContext chatContext) { + super.fillParseInfo(chatContext); parseInfo.setLimit(ORDERBY_MAX_RESULTS); parseInfo.setScore(parseInfo.getScore() + 2.0); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagListQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagListQuery.java index 26589e5c8..0c843e33b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagListQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagListQuery.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.query.rule.tag; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; @@ -12,16 +11,17 @@ import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.util.ContextUtils; +import org.apache.commons.collections.CollectionUtils; + import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; public abstract class TagListQuery extends TagSemanticQuery { @Override - public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { - super.fillParseInfo(modelId, queryContext, chatContext); + public void fillParseInfo(ChatContext chatContext) { + super.fillParseInfo(chatContext); this.addEntityDetailAndOrderByMetric(parseInfo); } @@ -29,13 +29,12 @@ public abstract class TagListQuery extends TagSemanticQuery { Long modelId = parseInfo.getModelId(); if (Objects.nonNull(modelId) && modelId > 0L) { ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(parseInfo.getModelId()); + ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(modelId); SemanticService schemaService = ContextUtils.getBean(SemanticService.class); - ModelSchema modelSchema = schemaService.getModelSchema(modelId); - + ModelSchema modelSchema = schemaService.getModelSchema(parseInfo.getModelId()); if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null && Objects.nonNull(modelSchema) && Objects.nonNull(modelSchema.getEntity())) { - Set dimensions = new LinkedHashSet(); + Set dimensions = new LinkedHashSet<>(); Set metrics = new LinkedHashSet(); Set orders = new LinkedHashSet(); ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc @@ -52,9 +51,7 @@ public abstract class TagListQuery extends TagSemanticQuery { chatDefaultConfig.getDimensions().stream() .forEach(dimension -> dimensions.add(dimension)); } - } - parseInfo.setDimensions(dimensions); parseInfo.setMetrics(metrics); parseInfo.setOrders(orders); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagSemanticQuery.java index aed3be716..13cd5ea69 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/tag/TagSemanticQuery.java @@ -1,12 +1,7 @@ package com.tencent.supersonic.chat.query.rule.tag; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; @@ -15,13 +10,19 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; @Slf4j public abstract class TagSemanticQuery extends RuleSemanticQuery { @@ -78,8 +79,8 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery { } @Override - public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { - super.fillParseInfo(modelId, queryContext, chatContext); + public void fillParseInfo(ChatContext chatContext) { + super.fillParseInfo(chatContext); parseInfo.setQueryType(QueryType.TAG); parseInfo.setLimit(TAG_MAX_RESULTS); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java index 59194a69f..377792bd9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java @@ -14,7 +14,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder { @Override public void fillResponse(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { - if (semanticParseInfo == null || semanticParseInfo.getModelId() <= 0L) { + if (semanticParseInfo == null) { return; } String queryMode = semanticParseInfo.getQueryMode(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/SimilarMetricExecuteResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/SimilarMetricExecuteResponder.java index e83c95f70..b8e73bc6f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/SimilarMetricExecuteResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/SimilarMetricExecuteResponder.java @@ -39,7 +39,7 @@ public class SimilarMetricExecuteResponder implements ExecuteResponder { } List metricNames = Collections.singletonList(parseInfo.getMetrics().iterator().next().getName()); Map filterCondition = new HashMap<>(); - filterCondition.put("modelId", parseInfo.getModelId().toString()); + filterCondition.put("modelId", parseInfo.getMetrics().iterator().next().getModel().toString()); filterCondition.put("type", SchemaElementType.METRIC.name()); RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames) .filterCondition(filterCondition).queryEmbeddings(null).build(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/QueryRankParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/QueryRankParseResponder.java index 53d52bbb8..5eab91552 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/QueryRankParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/QueryRankParseResponder.java @@ -3,22 +3,80 @@ package com.tencent.supersonic.chat.responder.parse; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.query.QueryRanker; -import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * Rank queries by score. */ +@Slf4j public class QueryRankParseResponder implements ParseResponder { + private static final int candidateTopSize = 5; + @Override public void fillResponse(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { List candidateQueries = queryContext.getCandidateQueries(); - QueryRanker queryRanker = ContextUtils.getBean(QueryRanker.class); - candidateQueries = queryRanker.rank(candidateQueries); + candidateQueries = rank(candidateQueries); queryContext.setCandidateQueries(candidateQueries); } + + public List rank(List candidateQueries) { + log.debug("pick before [{}]", candidateQueries); + if (CollectionUtils.isEmpty(candidateQueries)) { + return candidateQueries; + } + List selectedQueries = new ArrayList<>(); + if (candidateQueries.size() == 1) { + selectedQueries.addAll(candidateQueries); + } else { + selectedQueries = getTopCandidateQuery(candidateQueries); + } + generateParseInfoId(selectedQueries); + log.debug("pick after [{}]", selectedQueries); + return selectedQueries; + } + + public List getTopCandidateQuery(List semanticQueries) { + return semanticQueries.stream() + .filter(query -> !checkFullyInherited(query)) + .sorted((o1, o2) -> { + if (o1.getParseInfo().getScore() < o2.getParseInfo().getScore()) { + return 1; + } else if (o1.getParseInfo().getScore() > o2.getParseInfo().getScore()) { + return -1; + } + return 0; + }).limit(candidateTopSize) + .collect(Collectors.toList()); + } + + private void generateParseInfoId(List semanticQueries) { + for (int i = 0; i < semanticQueries.size(); i++) { + SemanticQuery query = semanticQueries.get(i); + query.getParseInfo().setId(i + 1); + } + } + + private boolean checkFullyInherited(SemanticQuery query) { + SemanticParseInfo parseInfo = query.getParseInfo(); + if (!(query instanceof RuleSemanticQuery)) { + return false; + } + + for (SchemaElementMatch match : parseInfo.getElementMatches()) { + if (!match.isInherited()) { + return false; + } + } + return parseInfo.getDateInfo() == null || parseInfo.getDateInfo().isInherited(); + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java index d10497e7c..2086787b6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; @@ -17,12 +18,6 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.chat.service.ConfigService; - -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -32,6 +27,10 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; + @RestController @RequestMapping({"/api/chat/conf", "/openapi/chat/conf"}) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java index 2957a8118..1921e0a55 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java @@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import java.util.List; +import java.util.Set; public interface ChatService { @@ -22,7 +23,7 @@ public interface ChatService { * @param chatId * @return */ - Long getContextModel(Integer chatId); + Set getContextModel(Integer chatId); ChatContext getOrCreateContext(int chatId); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ParseInfoService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ParseInfoService.java deleted file mode 100644 index 1a1b2614f..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ParseInfoService.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.tencent.supersonic.chat.service; - -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; - -public interface ParseInfoService { - - void updateParseInfo(SemanticParseInfo parseInfo); - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java index 69176eda8..7deeaae1f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java @@ -1,26 +1,14 @@ package com.tencent.supersonic.chat.service; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ModelSchema; -import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; @@ -35,7 +23,9 @@ import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.RatioOverType; @@ -44,6 +34,13 @@ import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; @@ -63,12 +60,16 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; @Service @Slf4j @@ -83,45 +84,38 @@ public class SemanticService { private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - public ModelSchema getModelSchema(Long id) { - ModelSchema modelSchema = schemaService.getModelSchema(id); - if (!Objects.isNull(modelSchema) && !Objects.isNull(modelSchema.getModel())) { - ChatConfigResp chaConfigInfo = - configService.fetchConfigByModelId(modelSchema.getModel().getId()); - // filter dimensions in blacklist - filterBlackDim(modelSchema, chaConfigInfo); - // filter metrics in blacklist - filterBlackMetric(modelSchema, chaConfigInfo); - } + public SemanticSchema getSemanticSchema() { + return schemaService.getSemanticSchema(); + } - return modelSchema; + public ModelSchema getModelSchema(Long id) { + return schemaService.getModelSchema(id); } public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) { if (parseInfo != null && parseInfo.getModelId() > 0) { EntityInfo entityInfo = getEntityInfo(parseInfo.getModelId()); - if (parseInfo.getDimensionFilters().size() <= 0) { + if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getModelInfo() == null) { entityInfo.setMetrics(null); entityInfo.setDimensions(null); return entityInfo; } - if (entityInfo.getModelInfo() != null && entityInfo.getModelInfo().getPrimaryEntityBizName() != null) { - String modelInfoPrimaryName = entityInfo.getModelInfo().getPrimaryEntityBizName(); + String primaryKey = entityInfo.getModelInfo().getPrimaryKey(); + if (StringUtils.isNotBlank(primaryKey)) { String modelInfoId = ""; for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName() - .equals(modelInfoPrimaryName)) { + .equals(primaryKey)) { if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { modelInfoId = chatFilter.getValue().toString(); } } } try { - setMainModel(entityInfo, parseInfo.getModelId(), - modelInfoId, user); + setMainModel(entityInfo, parseInfo, modelInfoId, user); return entityInfo; } catch (Exception e) { - log.error("setMainModel error {}", e); + log.error("setMainModel error", e); } } } @@ -152,8 +146,7 @@ public class SemanticService { modelInfo.setWords(modelSchema.getModel().getAlias()); modelInfo.setBizName(modelSchema.getModel().getBizName()); if (Objects.nonNull(modelSchema.getEntity())) { - modelInfo.setPrimaryEntityName(modelSchema.getEntity().getName()); - modelInfo.setPrimaryEntityBizName(modelSchema.getEntity().getBizName()); + modelInfo.setPrimaryKey(modelSchema.getEntity().getBizName()); } entityInfo.setModelInfo(modelInfo); @@ -190,21 +183,14 @@ public class SemanticService { return entityInfo; } - public String getPrimaryEntityBizName(EntityInfo entityInfo) { - if (Objects.isNull(entityInfo) || Objects.isNull(entityInfo.getModelInfo())) { - return null; - } - return entityInfo.getModelInfo().getPrimaryEntityBizName(); - } - - public void setMainModel(EntityInfo modelInfo, Long model, String entity, User user) { + public void setMainModel(EntityInfo modelInfo, SemanticParseInfo parseInfo, String entity, User user) { if (StringUtils.isEmpty(entity)) { return; } List entities = Collections.singletonList(entity); - QueryResultWithSchemaResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, model, entities, + QueryResultWithSchemaResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities, user); if (queryResultWithColumns != null) { @@ -225,15 +211,15 @@ public class SemanticService { } } - public QueryResultWithSchemaResp getQueryResultWithSchemaResp(EntityInfo modelInfo, Long model, + public QueryResultWithSchemaResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo, List entities, User user) { if (CollectionUtils.isEmpty(entities)) { return null; } - ModelSchema modelSchema = schemaService.getModelSchema(model); + ModelSchema modelSchema = schemaService.getModelSchema(parseInfo.getModelId()); modelInfo.setEntityId(entities.get(0)); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - semanticParseInfo.setModel(modelSchema.getModel()); + semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(parseInfo.getModelId()))); semanticParseInfo.setQueryType(QueryType.TAG); semanticParseInfo.setMetrics(getMetrics(modelInfo)); semanticParseInfo.setDimensions(getDimensions(modelInfo)); @@ -314,54 +300,7 @@ public class SemanticService { } private String getEntityPrimaryName(EntityInfo modelInfo) { - return modelInfo.getModelInfo().getPrimaryEntityBizName(); - } - - private void filterBlackMetric(ModelSchema modelSchema, ChatConfigResp chaConfigInfo) { - ItemVisibility visibility = generateFinalVisibility(chaConfigInfo); - if (Objects.nonNull(chaConfigInfo) && Objects.nonNull(visibility) - && !CollectionUtils.isEmpty(visibility.getBlackMetricIdList()) - && !CollectionUtils.isEmpty(modelSchema.getMetrics())) { - Set metric4Chat = modelSchema.getMetrics().stream() - .filter(metric -> !visibility.getBlackMetricIdList().contains(metric.getId())) - .collect(Collectors.toSet()); - modelSchema.setMetrics(metric4Chat); - } - } - - private void filterBlackDim(ModelSchema modelSchema, ChatConfigResp chatConfigInfo) { - ItemVisibility visibility = generateFinalVisibility(chatConfigInfo); - if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(visibility) - && !CollectionUtils.isEmpty(visibility.getBlackDimIdList()) - && !CollectionUtils.isEmpty(modelSchema.getDimensions())) { - Set dim4Chat = modelSchema.getDimensions().stream() - .filter(dim -> !visibility.getBlackDimIdList().contains(dim.getId())) - .collect(Collectors.toSet()); - modelSchema.setDimensions(dim4Chat); - } - } - - private ItemVisibility generateFinalVisibility(ChatConfigResp chatConfigInfo) { - ItemVisibility visibility = new ItemVisibility(); - - ChatAggConfigReq chatAggConfig = chatConfigInfo.getChatAggConfig(); - ChatDetailConfigReq chatDetailConfig = chatConfigInfo.getChatDetailConfig(); - - // both black is exist - if (Objects.nonNull(chatAggConfig) && Objects.nonNull(chatAggConfig.getVisibility()) - && Objects.nonNull(chatDetailConfig) && Objects.nonNull(chatDetailConfig.getVisibility())) { - List blackDimIdList = new ArrayList<>(); - blackDimIdList.addAll(chatAggConfig.getVisibility().getBlackDimIdList()); - blackDimIdList.retainAll(chatDetailConfig.getVisibility().getBlackDimIdList()); - List blackMetricIdList = new ArrayList<>(); - - blackMetricIdList.addAll(chatAggConfig.getVisibility().getBlackMetricIdList()); - blackMetricIdList.retainAll(chatDetailConfig.getVisibility().getBlackMetricIdList()); - - visibility.setBlackDimIdList(blackDimIdList); - visibility.setBlackMetricIdList(blackMetricIdList); - } - return visibility; + return modelInfo.getModelInfo().getPrimaryKey(); } public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java index 4eb2de5ec..89f07e8b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java @@ -4,8 +4,10 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; @@ -13,11 +15,18 @@ import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.QueryDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResp; -import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.persistence.repository.ChatRepository; +import com.tencent.supersonic.chat.service.ChatService; +import com.tencent.supersonic.chat.utils.SolvedQueryManager; +import com.tencent.supersonic.common.util.JsonUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.utils.Lists; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Comparator; @@ -28,14 +37,6 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import com.tencent.supersonic.chat.service.ChatService; -import com.tencent.supersonic.chat.utils.SolvedQueryManager; -import com.tencent.supersonic.common.util.JsonUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.compress.utils.Lists; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service("ChatService") @Primary @@ -56,7 +57,7 @@ public class ChatServiceImpl implements ChatService { } @Override - public Long getContextModel(Integer chatId) { + public Set getContextModel(Integer chatId) { if (Objects.isNull(chatId)) { return null; } @@ -65,8 +66,8 @@ public class ChatServiceImpl implements ChatService { return null; } SemanticParseInfo originalSemanticParse = chatContext.getParseInfo(); - if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getModelId())) { - return originalSemanticParse.getModelId(); + if (Objects.nonNull(originalSemanticParse) && Objects.nonNull(originalSemanticParse.getModel().getModelIds())) { + return originalSemanticParse.getModel().getModelIds(); } return null; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java index 4bde4f722..c9bc0ab2f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java @@ -6,39 +6,31 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; -import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; import com.tencent.supersonic.chat.api.pojo.request.Entity; +import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; +import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.ChatAggRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatAggRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp; import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo; import com.tencent.supersonic.chat.config.ChatConfig; +import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.SemanticService; -import com.tencent.supersonic.chat.utils.ComponentFactory; -import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.utils.ChatConfigHelper; +import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.VisibilityEvent; import com.tencent.supersonic.common.util.JsonUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; @@ -52,6 +44,13 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + @Slf4j @Service @@ -83,7 +82,6 @@ public class ConfigServiceImpl implements ConfigService { public Long addConfig(ChatConfigBaseReq configBaseCmd, User user) { log.info("[create model extend] object:{}", JsonUtil.toString(configBaseCmd, true)); duplicateCheck(configBaseCmd.getModelId()); - permissionCheckLogic(configBaseCmd.getModelId(), user.getName()); ChatConfig chaConfig = chatConfigHelper.newChatConfig(configBaseCmd, user); Long id = chatConfigRepository.createConfig(chaConfig); applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); @@ -107,7 +105,6 @@ public class ConfigServiceImpl implements ConfigService { configEditCmd.getModelId())) { throw new RuntimeException("editConfig, id and modelId are not allowed to be empty at the same time"); } - permissionCheckLogic(configEditCmd.getModelId(), user.getName()); ChatConfig chaConfig = chatConfigHelper.editChatConfig(configEditCmd, user); chatConfigRepository.updateConfig(chaConfig); applicationEventPublisher.publishEvent(new VisibilityEvent(this, chaConfig)); @@ -164,14 +161,6 @@ public class ConfigServiceImpl implements ConfigService { return itemNameVisibility; } - /** - * model administrators have the right to modify related configuration information. - */ - private Boolean permissionCheckLogic(Long modelId, String staffName) { - // todo - return true; - } - @Override public List search(ChatConfigFilter filter, User user) { log.info("[search model extend] object:{}", JsonUtil.toString(filter, true)); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java deleted file mode 100644 index 740f1baee..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java +++ /dev/null @@ -1,205 +0,0 @@ - -package com.tencent.supersonic.chat.service.impl; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.QueryType; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; -import com.tencent.supersonic.chat.service.ParseInfoService; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.DateConf.DateMode; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.knowledge.service.SchemaService; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ParserInfoServiceImpl implements ParseInfoService { - - - public void updateParseInfo(SemanticParseInfo parseInfo) { - SqlInfo sqlInfo = parseInfo.getSqlInfo(); - String correctS2SQL = sqlInfo.getCorrectS2SQL(); - if (StringUtils.isBlank(correctS2SQL)) { - return; - } - // if S2SQL equals correctS2SQL, than not update the parseInfo. - if (correctS2SQL.equals(sqlInfo.getS2SQL())) { - return; - } - - List expressions = SqlParserSelectHelper.getFilterExpression(correctS2SQL); - //set dataInfo - try { - if (!CollectionUtils.isEmpty(expressions)) { - DateConf dateInfo = getDateInfo(expressions); - if (dateInfo != null && parseInfo.getDateInfo() == null) { - parseInfo.setDateInfo(dateInfo); - } - } - } catch (Exception e) { - log.error("set dateInfo error :", e); - } - - //set filter - try { - Map fieldNameToElement = getNameToElement(parseInfo.getModelId()); - List result = getDimensionFilter(fieldNameToElement, expressions); - parseInfo.getDimensionFilters().addAll(result); - } catch (Exception e) { - log.error("set dimensionFilter error :", e); - } - - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - - if (Objects.isNull(semanticSchema)) { - return; - } - List allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); - Set metrics = getElements(parseInfo.getModelId(), allFields, semanticSchema.getMetrics()); - parseInfo.setMetrics(metrics); - - if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) { - parseInfo.setQueryType(QueryType.METRIC); - List groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); - List groupByDimensions = getFieldsExceptDate(groupByFields); - parseInfo.setDimensions( - getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions())); - } else { - parseInfo.setQueryType(QueryType.TAG); - List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); - List selectDimensions = getFieldsExceptDate(selectFields); - parseInfo.setDimensions( - getElements(parseInfo.getModelId(), selectDimensions, semanticSchema.getDimensions())); - } - } - - - private Set getElements(Long modelId, List allFields, List elements) { - return elements.stream() - .filter(schemaElement -> modelId.equals(schemaElement.getModel()) - && allFields.contains(schemaElement.getName()) - ).collect(Collectors.toSet()); - } - - private List getFieldsExceptDate(List allFields) { - if (org.springframework.util.CollectionUtils.isEmpty(allFields)) { - return new ArrayList<>(); - } - return allFields.stream() - .filter(entry -> !TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(entry)) - .collect(Collectors.toList()); - } - - - private List getDimensionFilter(Map fieldNameToElement, - List fieldExpressions) { - List result = Lists.newArrayList(); - for (FieldExpression expression : fieldExpressions) { - QueryFilter dimensionFilter = new QueryFilter(); - dimensionFilter.setValue(expression.getFieldValue()); - SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName()); - if (Objects.isNull(schemaElement)) { - continue; - } - dimensionFilter.setName(schemaElement.getName()); - dimensionFilter.setBizName(schemaElement.getBizName()); - dimensionFilter.setElementID(schemaElement.getId()); - - FilterOperatorEnum operatorEnum = FilterOperatorEnum.getSqlOperator(expression.getOperator()); - dimensionFilter.setOperator(operatorEnum); - dimensionFilter.setFunction(expression.getFunction()); - result.add(dimensionFilter); - } - return result; - } - - private DateConf getDateInfo(List fieldExpressions) { - List dateExpressions = fieldExpressions.stream() - .filter(expression -> TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(expression.getFieldName())) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(dateExpressions)) { - return null; - } - DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateMode.BETWEEN); - FieldExpression firstExpression = dateExpressions.get(0); - - FilterOperatorEnum firstOperator = FilterOperatorEnum.getSqlOperator(firstExpression.getOperator()); - if (FilterOperatorEnum.EQUALS.equals(firstOperator) && Objects.nonNull(firstExpression.getFieldValue())) { - dateInfo.setStartDate(firstExpression.getFieldValue().toString()); - dateInfo.setEndDate(firstExpression.getFieldValue().toString()); - dateInfo.setDateMode(DateMode.BETWEEN); - return dateInfo; - } - if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN, - FilterOperatorEnum.GREATER_THAN_EQUALS)) { - dateInfo.setStartDate(firstExpression.getFieldValue().toString()); - if (hasSecondDate(dateExpressions)) { - dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString()); - } - } - if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN, - FilterOperatorEnum.MINOR_THAN_EQUALS)) { - dateInfo.setEndDate(firstExpression.getFieldValue().toString()); - if (hasSecondDate(dateExpressions)) { - dateInfo.setStartDate(dateExpressions.get(1).getFieldValue().toString()); - } - } - return dateInfo; - } - - private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator, - FilterOperatorEnum... operatorEnums) { - return (Arrays.asList(operatorEnums).contains(firstOperator) && Objects.nonNull(expression.getFieldValue())); - } - - private boolean hasSecondDate(List dateExpressions) { - return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue()); - } - - protected Map getNameToElement(Long modelId) { - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - List dimensions = semanticSchema.getDimensions(); - List metrics = semanticSchema.getMetrics(); - - List allElements = Lists.newArrayList(); - allElements.addAll(dimensions); - allElements.addAll(metrics); - //support alias - return allElements.stream() - .filter(schemaElement -> schemaElement.getModel().equals(modelId)) - .flatMap(schemaElement -> { - Set> result = new HashSet<>(); - result.add(Pair.of(schemaElement.getName(), schemaElement)); - List aliasList = schemaElement.getAlias(); - if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) { - for (String alias : aliasList) { - result.add(Pair.of(alias, schemaElement)); - } - } - return result.stream(); - }) - .collect(Collectors.toMap(pair -> pair.getLeft(), pair -> pair.getRight(), (value1, value2) -> value2)); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java index 49add3b0d..4cc75ab45 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java @@ -14,9 +14,15 @@ import com.tencent.supersonic.chat.plugin.event.PluginDelEvent; import com.tencent.supersonic.chat.plugin.event.PluginUpdateEvent; import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.utils.ComponentFactory; -import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.util.Arrays; import java.util.Date; @@ -25,13 +31,6 @@ import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - @Service @Slf4j public class PluginServiceImpl implements PluginService { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 0c89ccc5b..ff9415281 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -252,7 +252,7 @@ public class QueryServiceImpl implements QueryService { solvedQueryManager.saveSolvedQuery(SolvedQueryReq.builder().parseId(queryReq.getParseId()) .queryId(queryReq.getQueryId()) .agentId(chatQueryDO.getAgentId()) - .modelId(parseInfo.getModelId()) + .modelId(parseInfo.getModelClusterKey()) .queryText(queryReq.getQueryText()).build()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java index 343fc2222..60e1a3b64 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java @@ -14,21 +14,27 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import com.tencent.supersonic.chat.mapper.MapperHelper; -import com.tencent.supersonic.chat.service.ConfigService; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.knowledge.dictionary.ModelInfoStat; -import com.tencent.supersonic.chat.mapper.ModelWithSemanticType; import com.tencent.supersonic.chat.mapper.MatchText; +import com.tencent.supersonic.chat.mapper.ModelWithSemanticType; import com.tencent.supersonic.chat.mapper.SearchMatchStrategy; import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.ChatService; +import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.SearchService; -import com.tencent.supersonic.knowledge.utils.NatureHelper; +import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.knowledge.dictionary.DictWord; import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.knowledge.dictionary.ModelInfoStat; import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.knowledge.utils.HanlpHelper; +import com.tencent.supersonic.knowledge.utils.NatureHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -41,11 +47,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; /** @@ -152,15 +153,13 @@ public class SearchServiceImpl implements SearchService { List possibleModels = NatureHelper.selectPossibleModels(originals); - Long contextModel = chatService.getContextModel(queryCtx.getChatId()); + Set contextModel = chatService.getContextModel(queryCtx.getChatId()); log.debug("possibleModels:{},modelStat:{},contextModel:{}", possibleModels, modelStat, contextModel); // If nothing is recognized or only metric are present, then add the contextModel. - if (nothingOrOnlyMetric(modelStat) && effectiveModel(contextModel)) { - List result = new ArrayList<>(); - result.add(contextModel); - return result; + if (nothingOrOnlyMetric(modelStat)) { + return contextModel.stream().filter(modelId -> modelId > 0).collect(Collectors.toList()); } return possibleModels; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java index 7f9b8173c..ba1bbad19 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java @@ -1,8 +1,5 @@ package com.tencent.supersonic.chat.utils; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; - import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ModelSchema; @@ -11,12 +8,20 @@ import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; -import com.tencent.supersonic.chat.config.Dim4Dict; import com.tencent.supersonic.chat.config.DefaultMetric; -import com.tencent.supersonic.chat.service.ConfigService; +import com.tencent.supersonic.chat.config.Dim4Dict; import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO; +import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode; import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; +import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -28,14 +33,8 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; -import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; @Component public class DictMetaHelper { @@ -202,7 +201,7 @@ public class DictMetaHelper { if (Objects.nonNull(dimIdAndRespPair) && dimIdAndRespPair.containsKey(dim4Dict.getDimId())) { String datasourceFilterSql = dimIdAndRespPair.get( - dim4Dict.getDimId()).getDatasourceFilterSql(); + dim4Dict.getDimId()).getModelFilterSql(); if (StringUtils.isNotEmpty(datasourceFilterSql)) { dim4Dict.getRuleList().add(datasourceFilterSql); } @@ -241,7 +240,7 @@ public class DictMetaHelper { PageInfo dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd); if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) { List list = dimensionPage.getList(); - return list.get(0).getDatasourceBizName(); + return list.get(0).getModelBizName(); } return ""; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ModelClusterBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ModelClusterBuilder.java new file mode 100644 index 000000000..218c58eb6 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ModelClusterBuilder.java @@ -0,0 +1,42 @@ +package com.tencent.supersonic.chat.utils; + + +import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; +import com.tencent.supersonic.common.pojo.ModelCluster; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class ModelClusterBuilder { + + public static List buildModelClusters(SemanticSchema semanticSchema) { + Map modelMap = semanticSchema.getModelSchemaMap(); + Set visited = new HashSet<>(); + List> modelClusters = new ArrayList<>(); + for (ModelSchema model : modelMap.values()) { + if (!visited.contains(model.getModel().getModel())) { + Set modelCluster = new HashSet<>(); + dfs(model, modelMap, visited, modelCluster); + modelClusters.add(modelCluster); + } + } + return modelClusters.stream().map(ModelCluster::build).collect(Collectors.toList()); + } + + private static void dfs(ModelSchema model, Map modelMap, + Set visited, Set modelCluster) { + visited.add(model.getModel().getModel()); + modelCluster.add(model.getModel().getModel()); + for (Long neighborId : model.getModelClusterSet()) { + if (!visited.contains(neighborId)) { + dfs(modelMap.get(neighborId), modelMap, visited, modelCluster); + } + } + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java index 2f7e81dfd..036de766c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java @@ -16,6 +16,12 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; + import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; @@ -25,18 +31,13 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; @Slf4j public class QueryReqBuilder { public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) { QueryStructReq queryStructCmd = new QueryStructReq(); - queryStructCmd.setModelId(parseInfo.getModelId()); + queryStructCmd.setModelIds(parseInfo.getModel().getModelIds()); queryStructCmd.setQueryType(parseInfo.getQueryType()); queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); @@ -128,15 +129,15 @@ public class QueryReqBuilder { * convert to QueryS2SQLReq * * @param querySql - * @param modelId + * @param modelIds * @return */ - public static QueryS2SQLReq buildS2SQLReq(String querySql, Long modelId) { + public static QueryS2SQLReq buildS2SQLReq(String querySql, Set modelIds) { QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); if (Objects.nonNull(querySql)) { queryS2SQLReq.setSql(querySql); } - queryS2SQLReq.setModelId(modelId); + queryS2SQLReq.setModelIds(modelIds); return queryS2SQLReq; } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessorTest.java index 65235316d..cb07b1abc 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessorTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessorTest.java @@ -6,8 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + import java.util.List; import java.util.Set; @@ -17,7 +20,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_necessaryDimension_groupBy() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 用户名"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT 用户名, sum(访问次数) FROM 超音数 GROUP BY 用户名"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); } @@ -26,7 +30,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_necessaryDimension_where() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 where 部门 = 'HR' group by 用户名"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT 用户名, sum(访问次数), count(DISTINCT 访问用户数) FROM 超音数 " + "WHERE 部门 = 'HR' GROUP BY 用户名"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); @@ -36,7 +41,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_dimensionNotDrillDown_groupBy() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 页面, 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面, 部门"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT 部门, sum(访问次数), count(DISTINCT 访问用户数) FROM 超音数 GROUP BY 部门"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); } @@ -45,7 +51,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_dimensionNotDrillDown_where() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 where 页面 = 'P1' group by 部门"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT 部门, sum(访问次数), count(DISTINCT 访问用户数) FROM 超音数 GROUP BY 部门"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); } @@ -54,7 +61,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_dimensionNotDrillDown_necessaryDimension() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 页面, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT sum(访问次数) FROM 超音数"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); } @@ -63,7 +71,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_dimensionDrillDown() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 用户名, 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 用户名, 部门"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, mockModelSchema()); + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema()); String expectedProcessedSql = "SELECT 用户名, 部门, sum(访问次数), count(DISTINCT 访问用户数) FROM 超音数 GROUP BY 用户名, 部门"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); } @@ -72,7 +81,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_noDrillDownDimensionSetting() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 页面, 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面, 用户名"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchemaNoDimensionSetting()); String expectedProcessedSql = "SELECT 页面, 用户名, sum(访问次数), count(DISTINCT 访问用户数) FROM 超音数 GROUP BY 页面, 用户名"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); @@ -82,7 +92,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_noDrillDownDimensionSetting_noAgg() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 访问次数 from 超音数"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchemaNoDimensionSetting()); String expectedProcessedSql = "select 访问次数 from 超音数"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); @@ -92,7 +103,8 @@ class MetricCheckPostProcessorTest { void testProcessCorrectSql_noDrillDownDimensionSetting_count() { MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor(); String correctSql = "select 部门, count(*) from 超音数 group by 部门"; - String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(correctSql, + SemanticParseInfo parseInfo = mockParseInfo(correctSql); + String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchemaNoDimensionSetting()); String expectedProcessedSql = "select 部门, count(*) from 超音数 group by 部门"; Assertions.assertEquals(expectedProcessedSql, actualProcessedSql); @@ -102,7 +114,7 @@ class MetricCheckPostProcessorTest { * 访问次数 drill down dimension is 用户名 and 部门 * 访问用户数 drill down dimension is 部门, and 部门 is necessary, 部门 need in select and group by or where expressions */ - private ModelSchema mockModelSchema() { + private SemanticSchema mockModelSchema() { ModelSchema modelSchema = new ModelSchema(); Set metrics = Sets.newHashSet( mockElement(1L, "访问次数", SchemaElementType.METRIC, @@ -113,10 +125,10 @@ class MetricCheckPostProcessorTest { ); modelSchema.setMetrics(metrics); modelSchema.setDimensions(mockDimensions()); - return modelSchema; + return new SemanticSchema(Lists.newArrayList(modelSchema)); } - private ModelSchema mockModelSchemaNoDimensionSetting() { + private SemanticSchema mockModelSchemaNoDimensionSetting() { ModelSchema modelSchema = new ModelSchema(); Set metrics = Sets.newHashSet( mockElement(1L, "访问次数", SchemaElementType.METRIC, Lists.newArrayList()), @@ -124,7 +136,7 @@ class MetricCheckPostProcessorTest { ); modelSchema.setMetrics(metrics); modelSchema.setDimensions(mockDimensions()); - return modelSchema; + return new SemanticSchema(Lists.newArrayList(modelSchema)); } private Set mockDimensions() { @@ -141,4 +153,10 @@ class MetricCheckPostProcessorTest { .relateSchemaElements(relateSchemaElements).build(); } + private SemanticParseInfo mockParseInfo(String correctSql) { + SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); + semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctSql); + return semanticParseInfo; + } + } \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java index aa06d458e..d4562f3fa 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java @@ -6,8 +6,8 @@ import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper; import com.tencent.supersonic.knowledge.semantic.RemoteSemanticInterpreter; import com.tencent.supersonic.chat.test.ChatBizLauncher; import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.query.service.QueryService; import org.junit.runner.RunWith; import org.slf4j.Logger; diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java index d30873460..aa9f28e5c 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java @@ -1,32 +1,24 @@ package com.tencent.supersonic.chat.test.context; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.when; - import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.config.DefaultMetric; import com.tencent.supersonic.chat.config.DefaultMetricInfo; import com.tencent.supersonic.chat.config.EntityInternalDetail; +import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper; import com.tencent.supersonic.chat.persistence.repository.impl.ChatContextRepositoryImpl; +import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.QueryService; +import com.tencent.supersonic.chat.service.impl.ConfigServiceImpl; +import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; -import com.tencent.supersonic.chat.service.impl.ConfigServiceImpl; -import com.tencent.supersonic.chat.service.ChatService; -import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper; -import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.MetricService; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - +import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import org.mockito.Mockito; @@ -34,6 +26,14 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.client.RestTemplate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.when; + @Configuration public class MockBeansConfiguration { diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java index 375a962c7..386a0c14d 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java @@ -17,19 +17,20 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.query.service.QueryService; import com.tencent.supersonic.semantic.query.service.SchemaService; -import java.util.HashMap; -import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.util.HashMap; +import java.util.List; + @Slf4j public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @@ -44,7 +45,7 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL()); - queryS2SQLReq.setModelId(queryStructReq.getModelId()); + queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet()); queryS2SQLReq.setVariables(new HashMap<>()); return queryByS2SQL(queryS2SQLReq, user); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java index 62a33490d..46769b7c2 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaValueMap; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; -import com.tencent.supersonic.semantic.api.model.pojo.Entity; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; @@ -20,8 +19,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -38,6 +35,7 @@ public class ModelSchemaBuilder { .alias(SchemaItem.getAliasList(resp.getAlias())) .build(); modelSchema.setModel(domain); + modelSchema.setModelRelas(resp.getModelRelas()); Set metrics = new HashSet<>(); for (MetricSchemaResp metric : resp.getMetrics()) { @@ -124,23 +122,19 @@ public class ModelSchemaBuilder { modelSchema.getDimensionValues().addAll(dimensionValues); modelSchema.getTags().addAll(tags); - Entity entity = resp.getEntity(); - if (Objects.nonNull(entity)) { - SchemaElement entityElement = new SchemaElement(); - - if (!CollectionUtils.isEmpty(entity.getNames()) && Objects.nonNull(entity.getEntityId())) { - Map idAndDimPair = dimensions.stream() - .collect( - Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement, (k1, k2) -> k2)); - if (idAndDimPair.containsKey(entity.getEntityId())) { - BeanUtils.copyProperties(idAndDimPair.get(entity.getEntityId()), entityElement); - entityElement.setType(SchemaElementType.ENTITY); - } - entityElement.setAlias(entity.getNames()); - modelSchema.setEntity(entityElement); - } + DimSchemaResp dim = resp.getPrimaryKey(); + if (dim != null) { + SchemaElement entity = SchemaElement.builder() + .model(resp.getId()) + .id(dim.getId()) + .name(dim.getName()) + .bizName(dim.getBizName()) + .type(SchemaElementType.ENTITY) + .useCnt(dim.getUseCnt()) + .alias(dim.getEntityAlias()) + .build(); + modelSchema.setEntity(entity); } - return modelSchema; } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java index c7aace663..191097cc7 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java @@ -1,10 +1,5 @@ package com.tencent.supersonic.knowledge.semantic; -import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER; - import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; @@ -31,15 +26,9 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; -import java.net.URI; -import java.net.URL; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -53,6 +42,18 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.net.URL; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Objects; + +import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER; + @Slf4j public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { @@ -73,7 +74,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL()); - queryS2SQLReq.setModelId(queryStructReq.getModelId()); + queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet()); queryS2SQLReq.setVariables(new HashMap<>()); return queryByS2SQL(queryS2SQLReq, user); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java b/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java new file mode 100644 index 000000000..e90c1d2ff --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/JoinCondition.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.common.pojo; + +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class JoinCondition { + + private String leftField; + + private String rightField; + + private FilterOperatorEnum operator; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java new file mode 100644 index 000000000..8bb33c28d --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelCluster.java @@ -0,0 +1,52 @@ +package com.tencent.supersonic.common.pojo; + +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Data +public class ModelCluster { + + private static final String split = "_"; + + private Set modelIds = new LinkedHashSet<>(); + + private String key; + + private String name; + + public static ModelCluster build(Set modelIds) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(modelIds); + modelCluster.setKey(StringUtils.join(modelIds, split)); + return modelCluster; + } + + public static ModelCluster build(String key) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(getModelIdFromKey(key)); + modelCluster.setKey(key); + return modelCluster; + } + + public void buildName(Map modelNameMap) { + name = modelNameMap.entrySet().stream().filter(entry -> + modelIds.contains(entry.getKey())).map(Map.Entry::getValue) + .collect(Collectors.joining(split)); + } + + public static Set getModelIdFromKey(String key) { + return Arrays.stream(key.split(split)) + .map(Long::parseLong).collect(Collectors.toSet()); + } + + public Long getFirstModel() { + return modelIds.stream().findFirst().orElse(null); + } + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java new file mode 100644 index 000000000..f1f2d415b --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ModelRela.java @@ -0,0 +1,24 @@ +package com.tencent.supersonic.common.pojo; + + +import lombok.Data; + +import java.util.List; + +@Data +public class ModelRela extends RecordInfo { + + private Long id; + + private Long domainId; + + private Long fromModelId; + + private Long toModelId; + + //left join, inner join, right join, outer join + private String joinType; + + private List joinConditions; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java index 7c4b10c83..cffccae76 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java @@ -42,4 +42,11 @@ public enum FilterOperatorEnum { } + public static boolean isValueCompare(FilterOperatorEnum filterOperatorEnum) { + return EQUALS.equals(filterOperatorEnum) || GREATER_THAN.equals(filterOperatorEnum) + || GREATER_THAN_EQUALS.equals(filterOperatorEnum) || MINOR_THAN.equals(filterOperatorEnum) + || MINOR_THAN_EQUALS.equals(filterOperatorEnum) || NOT_EQUALS.equals(filterOperatorEnum); + } + + } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java index 165e1e7cf..28c229c71 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java @@ -1,11 +1,5 @@ package com.tencent.supersonic.common.util.jsqlparser; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -35,6 +29,13 @@ import net.sf.jsqlparser.statement.select.SubSelect; import org.apache.commons.lang3.tuple.ImmutablePair; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + /** * Sql Parser Select Helper */ @@ -111,7 +112,7 @@ public class SqlParserSelectHelper { try { statement = CCJSqlParserUtil.parse(sql); } catch (JSQLParserException e) { - log.error("parse error", e); + log.error("parse error, sql:{}", sql, e); return null; } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index 6964309cb..5b5cec48b 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -40,6 +40,7 @@ import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; @Component @@ -109,12 +110,12 @@ public class ConfigureDemo implements ApplicationListener public void addDemoChatConfig_1() { ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(1L); + chatConfigBaseReq.setModelId(2L); ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = Arrays.asList(1L, 2L); - List metricIds0 = Arrays.asList(1L); + List dimensionIds0 = Collections.singletonList(3L); + List metricIds0 = Arrays.asList(1L, 2L); chatDefaultConfigDetail.setDimensionIds(dimensionIds0); chatDefaultConfigDetail.setMetricIds(metricIds0); chatDefaultConfigDetail.setUnit(7); @@ -126,8 +127,8 @@ public class ConfigureDemo implements ApplicationListener ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = Arrays.asList(1L, 2L); - List metricIds1 = Arrays.asList(1L); + List dimensionIds1 = Arrays.asList(3L); + List metricIds1 = Arrays.asList(1L, 2L); chatDefaultConfigAgg.setDimensionIds(dimensionIds1); chatDefaultConfigAgg.setMetricIds(metricIds1); chatDefaultConfigAgg.setUnit(7); @@ -138,34 +139,60 @@ public class ConfigureDemo implements ApplicationListener chatAggConfig.setVisibility(visibility1); List knowledgeInfos = new ArrayList<>(); KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); - knowledgeInfoReq.setItemId(1L); + knowledgeInfoReq.setItemId(3L); knowledgeInfoReq.setSearchEnable(true); knowledgeInfos.add(knowledgeInfoReq); - KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); - knowledgeInfoReq2.setItemId(2L); - knowledgeInfoReq2.setSearchEnable(true); - knowledgeInfos.add(knowledgeInfoReq2); chatAggConfig.setKnowledgeInfos(knowledgeInfos); chatConfigBaseReq.setChatAggConfig(chatAggConfig); - - List recommendedQuestions = new ArrayList<>(); - recommendedQuestions.add(new RecommendedQuestionReq("超音数访问次数")); - recommendedQuestions.add(new RecommendedQuestionReq("近15天超音数访问次数汇总")); - recommendedQuestions.add(new RecommendedQuestionReq("按部门统计超音数的访问人数")); - recommendedQuestions.add(new RecommendedQuestionReq("对比alice和lucy的停留时长")); - recommendedQuestions.add(new RecommendedQuestionReq("超音数访问次数最高的部门")); - chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); - configService.addConfig(chatConfigBaseReq, user); } public void addDemoChatConfig_2() { ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(2L); + chatConfigBaseReq.setModelId(3L); ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); - List dimensionIds0 = Arrays.asList(4L, 5L, 6L, 7L); + List dimensionIds0 = Arrays.asList(4L, 5L); + List metricIds0 = Arrays.asList(3L); + chatDefaultConfigDetail.setDimensionIds(dimensionIds0); + chatDefaultConfigDetail.setMetricIds(metricIds0); + chatDefaultConfigDetail.setUnit(7); + chatDefaultConfigDetail.setPeriod("DAY"); + chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); + ItemVisibility visibility0 = new ItemVisibility(); + chatDetailConfig.setVisibility(visibility0); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + + ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); + ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); + List dimensionIds1 = Arrays.asList(4L, 5L); + List metricIds1 = Arrays.asList(3L); + chatDefaultConfigAgg.setDimensionIds(dimensionIds1); + chatDefaultConfigAgg.setMetricIds(metricIds1); + chatDefaultConfigAgg.setUnit(7); + chatDefaultConfigAgg.setPeriod("DAY"); + chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); + chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); + ItemVisibility visibility1 = new ItemVisibility(); + chatAggConfig.setVisibility(visibility1); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setItemId(5L); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfos.add(knowledgeInfoReq); + chatAggConfig.setKnowledgeInfos(knowledgeInfos); + chatConfigBaseReq.setChatAggConfig(chatAggConfig); + configService.addConfig(chatConfigBaseReq, user); + } + + public void addDemoChatConfig_3() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setModelId(4L); + + ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); + ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); + List dimensionIds0 = Arrays.asList(6L, 7L, 8L, 9L); List metricIds0 = Arrays.asList(4L); chatDefaultConfigDetail.setDimensionIds(dimensionIds0); chatDefaultConfigDetail.setMetricIds(metricIds0); @@ -178,7 +205,7 @@ public class ConfigureDemo implements ApplicationListener ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); - List dimensionIds1 = Arrays.asList(4L, 5L, 6L, 7L); + List dimensionIds1 = Arrays.asList(6L, 7L, 8L, 9L); List metricIds1 = Arrays.asList(4L); chatDefaultConfigAgg.setDimensionIds(dimensionIds1); chatDefaultConfigAgg.setMetricIds(metricIds1); @@ -255,8 +282,8 @@ public class ConfigureDemo implements ApplicationListener private void addAgent2() { Agent agent = new Agent(); agent.setId(2); - agent.setName("圈实体"); - agent.setDescription("帮助您用自然语言圈选实体，支持多条件组合筛选"); + agent.setName("标签圈选"); + agent.setDescription("帮助您用自然语言进行圈选，支持多条件组合筛选"); agent.setStatus(1); agent.setEnableSearch(1); agent.setExamples(Lists.newArrayList("国风风格艺人", "港台地区的艺人", "风格为流行的艺人")); @@ -266,7 +293,7 @@ public class ConfigureDemo implements ApplicationListener ruleQueryTool.setType(AgentToolType.RULE); ruleQueryTool.setModelIds(Lists.newArrayList(-1L)); ruleQueryTool.setQueryModes(Lists.newArrayList( - "ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID")); + "TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID")); agentConfig.getTools().add(ruleQueryTool); LLMParserTool llmParserTool = new LLMParserTool(); @@ -310,6 +337,7 @@ public class ConfigureDemo implements ApplicationListener addSysParameter(); addDemoChatConfig_1(); addDemoChatConfig_2(); + addDemoChatConfig_3(); addPlugin_1(); addAgent1(); addAgent2(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java index 0dc73eb8b..cca2375e5 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java @@ -8,10 +8,9 @@ import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.request.DomainReq; import com.tencent.supersonic.semantic.api.model.request.ModelReq; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.ModelService; import lombok.extern.slf4j.Slf4j; @@ -20,6 +19,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -39,8 +39,6 @@ public class LoadBenchMarkDemo implements CommandLineRunner { private DomainService domainService; @Autowired private ModelService modelService; - @Autowired - private DatasourceService datasourceService; @Override public void run(String... args) { @@ -48,12 +46,11 @@ public class LoadBenchMarkDemo implements CommandLineRunner { return; } try { - addDomain(); - addModel_1(); - addDatasource_1(); - addDatasource_2(); - addDatasource_3(); - addDatasource_4(); + //addDomain(); + //addModel_1(); + //addModel_2(); + //addModel_3(); + //addModel_4(); } catch (Exception e) { log.error("Failed to add bench mark demo data", e); } @@ -72,105 +69,97 @@ public class LoadBenchMarkDemo implements CommandLineRunner { domainService.createDomain(domainReq, user); } - public void addModel_1() { + public void addModel_1() throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setName("测评数据_音乐"); - modelReq.setBizName("music"); - modelReq.setDomainId(2L); + modelReq.setDomainId(3L); + modelReq.setName("艺术类型"); + modelReq.setBizName("genre"); + modelReq.setDescription("艺术类型"); + modelReq.setDatabaseId(1L); modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); modelReq.setViewOrgs(Collections.singletonList("admin")); modelReq.setAdmins(Collections.singletonList("admin")); modelReq.setAdminOrgs(Collections.emptyList()); - modelService.createModel(modelReq, user); - } - - public void addDatasource_1() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setModelId(3L); - datasourceReq.setName("艺术类型"); - datasourceReq.setBizName("genre"); - datasourceReq.setDescription("艺术类型"); - datasourceReq.setDatabaseId(1L); - + ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); dimension1.setTypeParams(new DimensionTimeTypeParams()); dimensions.add(dimension1); dimensions.add(new Dim("活跃区域", "most_popular_in", DimensionTypeEnum.categorical.name(), 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.primary.name(), "g_name")); - datasourceReq.setIdentifiers(identifiers); + modelDetail.setIdentifiers(identifiers); List measures = new ArrayList<>(); Measure measure = new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 0); measures.add(measure); - datasourceReq.setMeasures(measures); + modelDetail.setMeasures(measures); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre"); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setQueryType("sql_query"); + modelDetail.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre"); + modelService.createModel(modelReq, user); } - public void addDatasource_2() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setModelId(3L); - datasourceReq.setName("艺术家"); - datasourceReq.setBizName("artist"); - datasourceReq.setDescription("艺术家"); - datasourceReq.setDatabaseId(1L); - + public void addModel_2() throws Exception { + ModelReq modelReq = new ModelReq(); + modelReq.setDomainId(3L); + modelReq.setName("艺术家"); + modelReq.setBizName("artist"); + modelReq.setDescription("艺术家"); + modelReq.setDatabaseId(1L); + ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); dimensions.add(new Dim("国籍", "country", DimensionTypeEnum.categorical.name(), 1)); dimensions.add(new Dim("性别", "gender", DimensionTypeEnum.categorical.name(), 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.primary.name(), "artist_name")); identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.foreign.name(), "g_name")); - datasourceReq.setIdentifiers(identifiers); + modelDetail.setIdentifiers(identifiers); - datasourceReq.setMeasures(Collections.emptyList()); + modelDetail.setMeasures(Collections.emptyList()); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist"); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setQueryType("sql_query"); + modelDetail.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist"); + modelService.createModel(modelReq, user); } - public void addDatasource_3() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setModelId(3L); - datasourceReq.setName("文件"); - datasourceReq.setBizName("files"); - datasourceReq.setDescription("文件"); - datasourceReq.setDatabaseId(1L); - + public void addModel_3() throws Exception { + ModelReq modelReq = new ModelReq(); + modelReq.setDomainId(3L); + modelReq.setName("文件"); + modelReq.setBizName("files"); + modelReq.setDescription("文件"); + modelReq.setDatabaseId(1L); + ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); dimensions.add(new Dim("持续时间", "duration", DimensionTypeEnum.categorical.name(), 1)); dimensions.add(new Dim("文件格式", "formats", DimensionTypeEnum.categorical.name(), 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.primary.name(), "f_id")); identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name")); - datasourceReq.setIdentifiers(identifiers); + modelDetail.setIdentifiers(identifiers); - datasourceReq.setMeasures(Collections.emptyList()); + modelDetail.setMeasures(Collections.emptyList()); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files"); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setQueryType("sql_query"); + modelDetail.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files"); + modelService.createModel(modelReq, user); } - public void addDatasource_4() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setModelId(3L); - datasourceReq.setName("歌曲"); - datasourceReq.setBizName("song"); - datasourceReq.setDescription("歌曲"); - datasourceReq.setDatabaseId(1L); - + public void addModel_4() throws Exception { + ModelReq modelReq = new ModelReq(); + modelReq.setDomainId(3L); + modelReq.setName("歌曲"); + modelReq.setBizName("song"); + modelReq.setDescription("歌曲"); + modelReq.setDatabaseId(1L); + ModelDetail modelDetail = new ModelDetail(); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); dimension1.setTypeParams(new DimensionTimeTypeParams()); @@ -178,22 +167,22 @@ public class LoadBenchMarkDemo implements CommandLineRunner { dimensions.add(new Dim("国家", "country", DimensionTypeEnum.categorical.name(), 1)); dimensions.add(new Dim("语种", "languages", DimensionTypeEnum.categorical.name(), 1)); dimensions.add(new Dim("发行时间", "releasedate", DimensionTypeEnum.categorical.name(), 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List identifiers = new ArrayList<>(); identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name")); identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id")); - datasourceReq.setIdentifiers(identifiers); + modelDetail.setIdentifiers(identifiers); List measures = new ArrayList<>(); measures.add(new Measure("分辨率", "resolution", AggOperatorEnum.SUM.name(), 1)); measures.add(new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 1)); - datasourceReq.setMeasures(measures); + modelDetail.setMeasures(measures); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, " + modelDetail.setQueryType("sql_query"); + modelDetail.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, " + " rating, languages, releasedate, resolution FROM song"); - datasourceService.createDatasource(datasourceReq, user); + modelService.createModel(modelReq, user); } } \ No newline at end of file diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java index dd524348d..c8918fefa 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java @@ -5,30 +5,33 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule; import com.tencent.supersonic.auth.api.authorization.service.AuthService; +import com.tencent.supersonic.common.pojo.JoinCondition; +import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.api.model.enums.DataTypeEnum; import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum; import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum; import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.semantic.api.model.pojo.Entity; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.request.DatabaseReq; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; import com.tencent.supersonic.semantic.api.model.request.DomainReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; import com.tencent.supersonic.semantic.model.domain.ModelService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -36,6 +39,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -58,7 +62,7 @@ public class LoadModelDataDemo implements CommandLineRunner { @Autowired private ModelService modelService; @Autowired - private DatasourceService datasourceService; + private ModelRelaService modelRelaService; @Autowired private DimensionService dimensionService; @Autowired @@ -75,11 +79,12 @@ public class LoadModelDataDemo implements CommandLineRunner { addDatabase(); addDomain(); addModel_1(); - addDatasource_1(); - addDatasource_2(); - addDatasource_3(); addModel_2(); - addDatasource_4(); + addModel_3(); + addModelRela_1(); + addModelRela_2(); + addDomain_2(); + addModel_4(); updateDimension(); updateMetric(); addAuthGroup_1(); @@ -106,6 +111,7 @@ public class LoadModelDataDemo implements CommandLineRunner { domainReq.setName("超音数"); domainReq.setBizName("supersonic"); domainReq.setParentId(0L); + domainReq.setStatus(StatusEnum.ONLINE.getCode()); domainReq.setViewers(Arrays.asList("admin", "tom", "jack")); domainReq.setViewOrgs(Collections.singletonList("admin")); domainReq.setAdmins(Collections.singletonList("admin")); @@ -113,51 +119,49 @@ public class LoadModelDataDemo implements CommandLineRunner { domainService.createDomain(domainReq, user); } - public void addModel_1() { + public void addModel_1() throws Exception { ModelReq modelReq = new ModelReq(); - modelReq.setName("超音数"); - modelReq.setBizName("supersonic"); + modelReq.setName("超音数用户部门"); + modelReq.setBizName("user_department"); + modelReq.setDescription("用户部门信息"); + modelReq.setDatabaseId(1L); modelReq.setDomainId(1L); modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); modelReq.setViewOrgs(Collections.singletonList("admin")); modelReq.setAdmins(Collections.singletonList("admin")); modelReq.setAdminOrgs(Collections.emptyList()); - modelService.createModel(modelReq, user); - } - - public void addDatasource_1() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setName("用户部门"); - datasourceReq.setBizName("user_department"); - datasourceReq.setDescription("用户部门"); - datasourceReq.setDatabaseId(1L); - + ModelDetail modelDetail = new ModelDetail(); List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); - datasourceReq.setIdentifiers(identifiers); + identifiers.add(new Identify("用户", IdentifyTypeEnum.primary.name(), "user_name")); + modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); dimensions.add(new Dim("部门", "department", DimensionTypeEnum.categorical.name(), 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); - datasourceReq.setMeasures(Collections.emptyList()); - datasourceReq.setQueryType("table_query"); - datasourceReq.setTableQuery("PUBLIC.s2_user_department"); - datasourceReq.setModelId(1L); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setMeasures(Collections.emptyList()); + modelDetail.setQueryType("table_query"); + modelDetail.setSqlQuery("select user_name,department from PUBLIC.s2_user_department"); + modelReq.setModelDetail(modelDetail); + modelReq.setDomainId(1L); + modelService.createModel(modelReq, user); } - public void addDatasource_2() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setName("PVUV统计"); - datasourceReq.setBizName("s2_pv_uv_statis"); - datasourceReq.setDescription("PVUV统计"); - datasourceReq.setDatabaseId(1L); - + public void addModel_2() throws Exception { + ModelReq modelReq = new ModelReq(); + modelReq.setName("超音数PVUV统计"); + modelReq.setBizName("s2_pv_uv_statis"); + modelReq.setDescription("超音数PVUV统计"); + modelReq.setDatabaseId(1L); + modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); + modelReq.setViewOrgs(Collections.singletonList("admin")); + modelReq.setAdmins(Collections.singletonList("admin")); + modelReq.setAdminOrgs(Collections.emptyList()); List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); - datasourceReq.setIdentifiers(identifiers); + ModelDetail modelDetail = new ModelDetail(); + identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "s2_pv_uv_statis_user_name")); + modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); @@ -166,7 +170,7 @@ public class LoadModelDataDemo implements CommandLineRunner { Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0); dimension2.setExpr("page"); dimensions.add(dimension2); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List measures = new ArrayList<>(); Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1); @@ -175,23 +179,29 @@ public class LoadModelDataDemo implements CommandLineRunner { Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); measures.add(measure2); - datasourceReq.setMeasures(measures); - datasourceReq.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setModelId(1L); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery("SELECT imp_date, user_name as s2_pv_uv_statis_user_name, page, 1 as pv, " + + "user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setQueryType("sql_query"); + modelReq.setDomainId(1L); + modelReq.setModelDetail(modelDetail); + modelService.createModel(modelReq, user); } - public void addDatasource_3() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setName("停留时长统计"); - datasourceReq.setBizName("s2_stay_time_statis"); - datasourceReq.setDescription("停留时长统计"); - datasourceReq.setDatabaseId(1L); - + public void addModel_3() throws Exception { + ModelReq modelReq = new ModelReq(); + modelReq.setName("停留时长统计"); + modelReq.setBizName("s2_stay_time_statis"); + modelReq.setDescription("停留时长统计"); + modelReq.setDatabaseId(1L); + modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); + modelReq.setViewOrgs(Collections.singletonList("admin")); + modelReq.setAdmins(Collections.singletonList("admin")); + modelReq.setAdminOrgs(Collections.emptyList()); List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); - datasourceReq.setIdentifiers(identifiers); + ModelDetail modelDetail = new ModelDetail(); + identifiers.add(new Identify("用户名称", IdentifyTypeEnum.primary.name(), "stay_hours_user_name")); + modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); @@ -200,44 +210,75 @@ public class LoadModelDataDemo implements CommandLineRunner { Dim dimension2 = new Dim("页面", "page", DimensionTypeEnum.categorical.name(), 1); dimension2.setExpr("page"); dimensions.add(dimension2); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); List measures = new ArrayList<>(); Measure measure1 = new Measure("停留时长", "stay_hours", AggregateTypeEnum.SUM.name(), 1); measures.add(measure1); - datasourceReq.setMeasures(measures); - datasourceReq.setTableQuery("PUBLIC.s2_stay_time_statis"); - datasourceReq.setQueryType("table_query"); - datasourceReq.setModelId(1L); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery( + "select imp_date,user_name as stay_hours_user_name,stay_hours,page from PUBLIC.s2_stay_time_statis"); + modelDetail.setQueryType("table_query"); + modelReq.setDomainId(1L); + modelReq.setModelDetail(modelDetail); + modelService.createModel(modelReq, user); } - public void addModel_2() { + public void addModelRela_1() { + List joinConditions = Lists.newArrayList(); + joinConditions.add(new JoinCondition("user_name", "s2_pv_uv_statis_user_name", FilterOperatorEnum.EQUALS)); + ModelRela modelRelaReq = new ModelRela(); + modelRelaReq.setDomainId(1L); + modelRelaReq.setFromModelId(1L); + modelRelaReq.setToModelId(2L); + modelRelaReq.setJoinType("left join"); + modelRelaReq.setJoinConditions(joinConditions); + modelRelaService.save(modelRelaReq, user); + } + + public void addModelRela_2() { + List joinConditions = Lists.newArrayList(); + joinConditions.add(new JoinCondition("user_name", "stay_hours_user_name", FilterOperatorEnum.EQUALS)); + ModelRela modelRelaReq = new ModelRela(); + modelRelaReq.setDomainId(1L); + modelRelaReq.setFromModelId(1L); + modelRelaReq.setToModelId(3L); + modelRelaReq.setJoinType("left join"); + modelRelaReq.setJoinConditions(joinConditions); + modelRelaService.save(modelRelaReq, user); + } + + public void addDomain_2() { + DomainReq domainReq = new DomainReq(); + domainReq.setName("艺人库"); + domainReq.setBizName("supersonic"); + domainReq.setParentId(0L); + domainReq.setStatus(StatusEnum.ONLINE.getCode()); + domainReq.setViewers(Arrays.asList("admin", "tom", "jack")); + domainReq.setViewOrgs(Collections.singletonList("admin")); + domainReq.setAdmins(Collections.singletonList("admin")); + domainReq.setAdminOrgs(Collections.emptyList()); + domainService.createDomain(domainReq, user); + } + + public void addModel_4() throws Exception { ModelReq modelReq = new ModelReq(); modelReq.setName("艺人库"); modelReq.setBizName("singer"); - modelReq.setDomainId(1L); + modelReq.setDescription("艺人库"); + modelReq.setDatabaseId(1L); + modelReq.setDomainId(2L); modelReq.setViewers(Arrays.asList("admin", "tom", "jack")); modelReq.setViewOrgs(Collections.singletonList("admin")); modelReq.setAdmins(Collections.singletonList("admin")); modelReq.setAdminOrgs(Collections.emptyList()); - modelReq.setEntity(new Entity(7L, Arrays.asList("歌手", "艺人"))); - modelService.createModel(modelReq, user); - } - - public void addDatasource_4() throws Exception { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setName("艺人库"); - datasourceReq.setBizName("singer"); - datasourceReq.setDescription("艺人库"); - datasourceReq.setDatabaseId(1L); - + ModelDetail modelDetail = new ModelDetail(); List identifiers = new ArrayList<>(); Identify identify = new Identify("歌手名", IdentifyTypeEnum.primary.name(), "singer_name"); identify.setEntityNames(Lists.newArrayList("歌手", "艺人")); identifiers.add(identify); - datasourceReq.setIdentifiers(identifiers); + modelDetail.setIdentifiers(identifiers); List dimensions = new ArrayList<>(); Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); @@ -249,26 +290,26 @@ public class LoadModelDataDemo implements CommandLineRunner { DimensionTypeEnum.categorical.name(), 1)); dimensions.add(new Dim("风格", "genre", DimensionTypeEnum.categorical.name(), 1, 1)); - datasourceReq.setDimensions(dimensions); + modelDetail.setDimensions(dimensions); Measure measure1 = new Measure("播放量", "js_play_cnt", "sum", 1); Measure measure2 = new Measure("下载量", "down_cnt", "sum", 1); Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1); - datasourceReq.setMeasures(Lists.newArrayList(measure1, measure2, measure3)); - datasourceReq.setQueryType("table_query"); - datasourceReq.setTableQuery("PUBLIC.singer"); - datasourceReq.setModelId(2L); - datasourceService.createDatasource(datasourceReq, user); + modelDetail.setMeasures(Lists.newArrayList(measure1, measure2, measure3)); + modelDetail.setQueryType("table_query"); + modelDetail.setTableQuery("PUBLIC.singer"); + modelReq.setModelDetail(modelDetail); + modelService.createModel(modelReq, user); } public void updateDimension() throws Exception { DimensionReq dimensionReq = new DimensionReq(); dimensionReq.setModelId(1L); dimensionReq.setType(DimensionTypeEnum.categorical.name()); - dimensionReq.setId(3L); + dimensionReq.setId(4L); dimensionReq.setName("页面"); dimensionReq.setBizName("page"); - dimensionReq.setDatasourceId(3L); + dimensionReq.setModelId(3L); dimensionReq.setAlias("page"); dimensionReq.setSemanticType(SemanticTypeEnum.CATEGORY.name()); dimensionReq.setSensitiveLevel(2); @@ -302,7 +343,7 @@ public class LoadModelDataDemo implements CommandLineRunner { public void addAuthGroup_1() { AuthGroup authGroupReq = new AuthGroup(); - authGroupReq.setModelId(1L); + authGroupReq.setModelId(3L); authGroupReq.setName("admin-permission"); List authRules = new ArrayList<>(); @@ -319,7 +360,7 @@ public class LoadModelDataDemo implements CommandLineRunner { public void addAuthGroup_2() { AuthGroup authGroupReq = new AuthGroup(); - authGroupReq.setModelId(1L); + authGroupReq.setModelId(3L); authGroupReq.setName("tom_sales_permission"); List authRules = new ArrayList<>(); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 379827202..fa70e16ab 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -3,7 +3,8 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\ com.tencent.supersonic.chat.mapper.HanlpDictMapper, \ com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \ com.tencent.supersonic.chat.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.mapper.EntityMapper + com.tencent.supersonic.chat.mapper.EntityMapper, \ + com.tencent.supersonic.chat.mapper.ModelClusterMapper com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \ @@ -17,7 +18,8 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.corrector.SelectCorrector, \ com.tencent.supersonic.chat.corrector.WhereCorrector, \ com.tencent.supersonic.chat.corrector.GroupByCorrector, \ - com.tencent.supersonic.chat.corrector.HavingCorrector + com.tencent.supersonic.chat.corrector.HavingCorrector, \ + com.tencent.supersonic.chat.corrector.FromCorrector com.tencent.supersonic.chat.llm.LLMInterpreter=\ com.tencent.supersonic.chat.llm.HttpLLMInterpreter diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt index 9871036d4..46dd21b05 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _2_7 100 -陈奕迅 _2_7 100 -林俊杰 _2_7 100 -张碧晨 _2_7 100 -程响 _2_7 100 -Taylor#Swift _2_7 100 -内地 _2_4 100 -欧美 _2_4 100 -港台 _2_4 100 -流行 _2_6 100 -国风 _2_6 100 \ No newline at end of file +周杰伦 _4_9 100 +陈奕迅 _4_9 100 +林俊杰 _4_9 100 +张碧晨 _4_9 100 +程响 _4_9 100 +Taylor#Swift _4_9 100 +内地 _4_6 100 +欧美 _4_6 100 +港台 _4_6 100 +流行 _4_8 100 +国风 _4_8 100 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt index f519c23fb..c80af2b98 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _1_3 52 -p2 _1_3 47 -p3 _1_3 31 -p4 _1_3 36 -p5 _1_3 50 -p6 _1_3 38 \ No newline at end of file +p1 _3_4 52 +p2 _3_4 47 +p3 _3_4 31 +p4 _3_4 36 +p5 _3_4 50 +p6 _3_4 38 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_2_7.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_2_7.txt deleted file mode 100644 index 6699a0fc8..000000000 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_2_7.txt +++ /dev/null @@ -1,9 +0,0 @@ -周杰伦 _2_7 9000 -周深 _2_7 8000 -周传雄 _2_7 7000 -周华建 _2_7 6000 -陈奕迅 _2_7 8000 -林俊杰 _2_7 7000 -张碧晨 _2_7 7000 -程响 _2_7 7000 -Taylor#Swift _2_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt new file mode 100644 index 000000000..ec8a5bd14 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt @@ -0,0 +1,9 @@ +周杰伦 _4_9 9000 +周深 _4_9 8000 +周传雄 _4_9 7000 +周华建 _4_9 6000 +陈奕迅 _4_9 8000 +林俊杰 _4_9 7000 +张碧晨 _4_9 7000 +程响 _4_9 7000 +Taylor#Swift _4_9 7000 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt index 6a647af87..e69de29bb 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt @@ -1,68 +0,0 @@ -tagore _3_8 9000 -nazrul _3_8 9000 -民间 _3_8 9000 -现代 _3_8 9000 -蓝调 _3_8 9000 -流行 _3_8 9000 -孟加拉国 _3_10 9000 -锡尔赫特、吉大港、库斯蒂亚 _3_10 9000 -加拿大 _3_10 9000 -美国 _3_10 9000 -Shrikanta _3_11 9000 -Prity _3_11 9000 -Farida _3_11 9000 -Topu _3_11 9000 -Enrique _3_11 9000 -Michel _3_11 9000 -孟加拉国 _3_12 9000 -印度 _3_12 9000 -美国 _3_12 9000 -英国 _3_12 9000 -男性 _3_13 9000 -女性 _3_13 9000 -mp4 _3_19 9000 -mp3 _3_19 9000 -Tumi#长袍#尼罗布 _3_20 9000 -舒克诺#帕塔尔#努普尔#帕埃 _3_20 9000 -阿米·奥帕尔·霍伊 _3_20 9000 -我的爱 _3_20 9000 -打败它 _3_20 9000 -阿杰伊阿卡什 _3_20 9000 -孟加拉国 _3_22 9000 -印度 _3_22 9000 -美国 _3_22 9000 -英国 _3_22 9000 -孟加拉语 _3_26 9000 -英文 _3_26 9000 -======= -孟加拉国 _3_8 9000 -锡尔赫特、吉大港、库斯蒂亚 _3_8 9000 -加拿大 _3_8 9000 -美国 _3_8 9000 -tagore _3_9 9000 -nazrul _3_9 9000 -民间 _3_9 9000 -现代 _3_9 9000 -蓝调 _3_9 9000 -流行 _3_9 9000 -孟加拉国 _3_10 9000 -印度 _3_10 9000 -美国 _3_10 9000 -英国 _3_10 9000 -男性 _3_11 9000 -女性 _3_11 9000 -Shrikanta _3_12 9000 -Prity _3_12 9000 -Farida _3_12 9000 -Topu _3_12 9000 -Enrique _3_12 9000 -Michel _3_12 9000 -mp4 _3_14 9000 -mp3 _3_14 9000 -孟加拉语 _3_16 9000 -英文 _3_16 9000 -Tumi#长袍#尼罗布 _3_18 9000 -舒克诺#帕塔尔#努普尔#帕埃 _3_18 9000 -阿米·奥帕尔·霍伊 _3_18 9000 -我的爱 _3_18 9000 -打败它 _3_18 9000 diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index f1c1195ad..a749e9462 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -133,6 +133,10 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `view_org` varchar(3000) DEFAULT NULL , -- domain available organization `entity` varchar(500) DEFAULT NULL , -- domain entity info `drill_down_dimensions` varchar(500) DEFAULT NULL , -- drill down dimensions info + `database_id` INT NOT NULL , + `model_detail` LONGVARCHAR NOT NULL , + `depends` varchar(500) DEFAULT NULL , + `filter_sql` varchar(1000) DEFAULT NULL , PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_model IS 'model information'; @@ -156,16 +160,12 @@ CREATE TABLE `s2_database` ( COMMENT ON TABLE s2_database IS 'database instance table'; CREATE TABLE IF NOT EXISTS `s2_datasource` ( - `id` INT NOT NULL AUTO_INCREMENT, - `model_id` INT NOT NULL , - `name` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) DEFAULT NULL , - `database_id` INT NOT NULL , - `datasource_detail` LONGVARCHAR NOT NULL , - `status` int(11) DEFAULT NULL , - `depends` varchar(500) DEFAULT NULL , - `filter_sql` varchar(1000) DEFAULT NULL , + `created_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , `updated_at` TIMESTAMP NOT NULL , @@ -208,7 +208,6 @@ COMMENT ON TABLE s2_metric IS 'metric information table'; CREATE TABLE IF NOT EXISTS `s2_dimension` ( `id` INT NOT NULL AUTO_INCREMENT , `model_id` INT NOT NULL , - `datasource_id` INT NOT NULL , `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) NOT NULL , @@ -231,20 +230,16 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` ( ); COMMENT ON TABLE s2_dimension IS 'dimension information table'; -create table s2_datasource_rela +CREATE TABLE s2_model_rela ( - id INT AUTO_INCREMENT, - model_id INT null, - datasource_from INT null, - datasource_to INT null, - join_key varchar(100) null, - created_at TIMESTAMP null, - created_by varchar(100) null, - updated_at TIMESTAMP null, - updated_by varchar(100) null, + id BIGINT AUTO_INCREMENT, + domain_id BIGINT, + from_model_id BIGINT, + to_model_id BIGINT, + join_type VARCHAR(255), + join_condition VARCHAR(255), PRIMARY KEY (`id`) ); -COMMENT ON TABLE s2_datasource_rela IS 'data source association table'; create table s2_view_info ( @@ -535,4 +530,5 @@ CREATE TABLE s2_sys_parameter id INT PRIMARY KEY AUTO_INCREMENT, admin varchar(500), parameters text null -); \ No newline at end of file +); + diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 87ed2ecaf..71c195f2b 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -453,4 +453,14 @@ CREATE TABLE s2_sys_parameter id int primary key AUTO_INCREMENT COMMENT '主键id', admin varchar(500) COMMENT '系统管理员', parameters text null COMMENT '配置项' +); + +CREATE TABLE s2_model_rela +( + id bigint primary key AUTO_INCREMENT, + domain_id bigint, + from_model_id bigint, + to_model_id bigint, + join_type VARCHAR(255), + join_condition VARCHAR(255) ); \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 51769b52b..7f9cd6792 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -96,4 +96,21 @@ alter table s2_chat_config add column `llm_examples` text COMMENT 'llm examples' alter table s2_datasource add column `filter_sql` varchar(1000) COMMENT 'filter_sql' after depends; --20231120 -alter table s2_dimension add column `is_tag` int(10) DEFAULT NULL; \ No newline at end of file +alter table s2_dimension add column `is_tag` int(10) DEFAULT NULL; + +--20231125 +alter table s2_model add column `database_id` INT NOT NULL; +alter table s2_model add column `model_detail` text NOT NULL; +alter table s2_model add column `depends` varchar(500) DEFAULT NULL; +alter table s2_model add column `filter_sql` varchar(1000) DEFAULT NULL; + +CREATE TABLE s2_model_rela +( + id BIGINT AUTO_INCREMENT, + domain_id BIGINT, + from_model_id BIGINT, + to_model_id BIGINT, + join_type VARCHAR(255), + join_condition VARCHAR(255), + PRIMARY KEY (`id`) +); \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java index 54be0d19a..12ff3ff36 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java @@ -1,14 +1,7 @@ package com.tencent.supersonic.integration; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; -import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery; @@ -19,15 +12,17 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; +import org.junit.Assert; +import org.junit.Test; + import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.BeanUtils; + +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; public class MetricQueryTest extends BaseQueryTest { @@ -46,7 +41,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); + FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setQueryType(QueryType.METRIC); @@ -129,7 +124,7 @@ public class MetricQueryTest extends BaseQueryTest { List list = new ArrayList<>(); list.add("alice"); list.add("lucy"); - QueryFilter dimensionFilter = DataUtils.getFilter("user_name", FilterOperatorEnum.IN, list, "用户名", 2L); + QueryFilter dimensionFilter = DataUtils.getFilter("user_name", FilterOperatorEnum.IN, list, "用户", 2L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); @@ -150,7 +145,9 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.setAggType(SUM); expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); + expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名")); + expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名称")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY")); expectedParseInfo.setQueryType(QueryType.METRIC); @@ -195,7 +192,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", - FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); + FilterOperatorEnum.EQUALS, "alice", "用户", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); expectedParseInfo.setQueryType(QueryType.METRIC); @@ -203,44 +200,4 @@ public class MetricQueryTest extends BaseQueryTest { assertQueryResult(expectedResult, actualResult); } - @Test - public void queryTest_config_visibility() throws Exception { - // 1. round_1 use blacklist - ChatConfigResp chatConfig = configService.fetchConfigByModelId(1L); - ChatConfigEditReqReq extendEditCmd = new ChatConfigEditReqReq(); - BeanUtils.copyProperties(chatConfig, extendEditCmd); - // add blacklist - List blackMetrics = Arrays.asList(2L); - extendEditCmd.getChatAggConfig().getVisibility().setBlackMetricIdList(blackMetrics); - configService.editConfig(extendEditCmd, User.getFakeUser()); - - QueryResult actualResult = submitNewChat("超音数访问人数、访问次数"); - QueryResult expectedResult = new QueryResult(); - SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); - expectedResult.setChatContext(expectedParseInfo); - - expectedResult.setQueryMode(MetricModelQuery.QUERY_MODE); - expectedParseInfo.setAggType(NONE); - - expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setQueryType(QueryType.METRIC); - - assertQueryResult(expectedResult, actualResult); - - // 2. round_2 no blacklist - // remove blacklist - extendEditCmd.getChatAggConfig().setVisibility(new ItemVisibility()); - configService.editConfig(extendEditCmd, User.getFakeUser()); - - actualResult = submitNewChat("超音数访问人数、访问次数"); - expectedParseInfo.getMetrics().clear(); - expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问人数")); - - assertQueryResult(expectedResult, actualResult); - - } - } diff --git a/launchers/standalone/src/test/resources/META-INF/spring.factories b/launchers/standalone/src/test/resources/META-INF/spring.factories index 98a40f3b6..be224349b 100644 --- a/launchers/standalone/src/test/resources/META-INF/spring.factories +++ b/launchers/standalone/src/test/resources/META-INF/spring.factories @@ -3,7 +3,8 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\ com.tencent.supersonic.chat.mapper.HanlpDictMapper, \ com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \ com.tencent.supersonic.chat.mapper.QueryFilterMapper, \ - com.tencent.supersonic.chat.mapper.EntityMapper + com.tencent.supersonic.chat.mapper.EntityMapper, \ + com.tencent.supersonic.chat.mapper.ModelClusterMapper com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \ diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 9ace2760e..954d50137 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -80,21 +80,6 @@ CREATE TABLE IF NOT EXISTS `s2_chat_config` ( ) ; COMMENT ON TABLE s2_chat_config IS 'chat config information table '; -CREATE TABLE IF NOT EXISTS s2_agent -( - id int AUTO_INCREMENT, - name varchar(100) null, - description varchar(500) null, - status int null, - examples varchar(500) null, - config varchar(2000) null, - created_by varchar(100) null, - created_at TIMESTAMP null, - updated_by varchar(100) null, - updated_at TIMESTAMP null, - enable_search int null, - PRIMARY KEY (`id`) - ); COMMENT ON TABLE s2_agent IS 'agent information table'; create table s2_user ( @@ -134,8 +119,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `name` varchar(255) DEFAULT NULL , -- domain name `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID - `alias` varchar(255) DEFAULT NULL , -- alias name - `status` INT DEFAULT NULL , + `alias` varchar(255) DEFAULT NULL , -- internal name + `status` INT DEFAULT NULL, `description` varchar(500) DEFAULT NULL , `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , @@ -148,6 +133,10 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `view_org` varchar(3000) DEFAULT NULL , -- domain available organization `entity` varchar(500) DEFAULT NULL , -- domain entity info `drill_down_dimensions` varchar(500) DEFAULT NULL , -- drill down dimensions info + `database_id` INT NOT NULL , + `model_detail` LONGVARCHAR NOT NULL , + `depends` varchar(500) DEFAULT NULL , + `filter_sql` varchar(1000) DEFAULT NULL , PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_model IS 'model information'; @@ -171,16 +160,12 @@ CREATE TABLE `s2_database` ( COMMENT ON TABLE s2_database IS 'database instance table'; CREATE TABLE IF NOT EXISTS `s2_datasource` ( - `id` INT NOT NULL AUTO_INCREMENT, - `model_id` INT NOT NULL , - `name` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) DEFAULT NULL , - `database_id` INT NOT NULL , - `depends` varchar(500) DEFAULT NULL , - `datasource_detail` LONGVARCHAR NOT NULL , - `status` int(11) DEFAULT NULL , - `filter_sql` varchar(1000) DEFAULT NULL , + `created_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , `updated_at` TIMESTAMP NOT NULL , @@ -202,7 +187,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` ( `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) DEFAULT NULL , - `status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted + `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal `sensitive_level` INT NOT NULL , `type` varchar(50) NOT NULL , -- type proxy,expr `type_params` LONGVARCHAR DEFAULT NULL , @@ -223,11 +208,10 @@ COMMENT ON TABLE s2_metric IS 'metric information table'; CREATE TABLE IF NOT EXISTS `s2_dimension` ( `id` INT NOT NULL AUTO_INCREMENT , `model_id` INT NOT NULL , - `datasource_id` INT NOT NULL , `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) NOT NULL , - `status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted + `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal `sensitive_level` INT DEFAULT NULL , `data_type` varchar(50) DEFAULT NULL , -- type date,array,varchar `type` varchar(50) NOT NULL , -- type categorical,time @@ -246,20 +230,16 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` ( ); COMMENT ON TABLE s2_dimension IS 'dimension information table'; -create table s2_datasource_rela +CREATE TABLE s2_model_rela ( - id INT AUTO_INCREMENT, - model_id INT null, - datasource_from INT null, - datasource_to INT null, - join_key varchar(100) null, - created_at TIMESTAMP null, - created_by varchar(100) null, - updated_at TIMESTAMP null, - updated_by varchar(100) null, + id BIGINT AUTO_INCREMENT, + domain_id BIGINT, + from_model_id BIGINT, + to_model_id BIGINT, + join_type VARCHAR(255), + join_condition VARCHAR(255), PRIMARY KEY (`id`) ); -COMMENT ON TABLE s2_datasource_rela IS 'data source association table'; create table s2_view_info ( @@ -295,7 +275,6 @@ CREATE TABLE `s2_query_stat_info` ( `native_query` INT DEFAULT NULL, -- 1-detail query, 0-aggregation query `start_date` varchar(50) DEFAULT NULL, `end_date` varchar(50) DEFAULT NULL, - `query_opt_mode` varchar(50) DEFAULT NULL, `dimensions`LONGVARCHAR , -- dimensions involved in sql `metrics`LONGVARCHAR , -- metric involved in sql `select_cols`LONGVARCHAR , @@ -307,6 +286,7 @@ CREATE TABLE `s2_query_stat_info` ( `use_sql_cache` TINYINT DEFAULT '-1' , -- whether to hit the sql cache `sql_cache_key`LONGVARCHAR , -- sql cache key `result_cache_key`LONGVARCHAR , -- result cache key + `query_opt_mode` varchar(50) DEFAULT NULL , PRIMARY KEY (`id`) ) ; COMMENT ON TABLE s2_query_stat_info IS 'query statistics table'; @@ -370,6 +350,22 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_plugin IS 'plugin information table'; +CREATE TABLE IF NOT EXISTS s2_agent +( + id int AUTO_INCREMENT, + name varchar(100) null, + description varchar(500) null, + status int null, + examples varchar(500) null, + config varchar(2000) null, + created_by varchar(100) null, + created_at TIMESTAMP null, + updated_by varchar(100) null, + updated_at TIMESTAMP null, + enable_search int null, + PRIMARY KEY (`id`) +); COMMENT ON TABLE s2_agent IS 'agent information table'; + -------demo for semantic and chat CREATE TABLE IF NOT EXISTS `s2_user_department` ( @@ -405,8 +401,68 @@ CREATE TABLE IF NOT EXISTS `singer` ( ); COMMENT ON TABLE singer IS 'singer_info'; +CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( + `id` INT NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , -- task name + `description` varchar(255) , + `command`LONGVARCHAR NOT NULL , -- task Request Parameters + `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 + `status` INT NOT NULL , -- the final status of the task + `dimension_ids` varchar(500) NULL , + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `created_by` varchar(100) NOT NULL , + `progress` DOUBLE default 0.00 , -- task real-time progress + `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds + `message` LONGVARCHAR , -- remark related information + PRIMARY KEY (`id`) +); +COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; + +-- benchmark +CREATE TABLE IF NOT EXISTS `genre` ( + `g_name` varchar(20) NOT NULL , -- genre name + `rating` INT , + `most_popular_in` varchar(50) , + PRIMARY KEY (`g_name`) + ); +COMMENT ON TABLE genre IS 'genre'; + +CREATE TABLE IF NOT EXISTS `artist` ( + `artist_name` varchar(50) NOT NULL , -- genre name + `country` varchar(20) , + `gender` varchar(20) , + `g_name` varchar(50) + ); +COMMENT ON TABLE artist IS 'artist'; + +CREATE TABLE IF NOT EXISTS `files` ( + `f_id` bigINT NOT NULL, + `artist_name` varchar(50) , + `file_size` varchar(20) , + `duration` varchar(20) , + `formats` varchar(20) , + PRIMARY KEY (`f_id`) + ); +COMMENT ON TABLE files IS 'files'; + +CREATE TABLE IF NOT EXISTS `song` ( + `imp_date` varchar(50) , + `song_name` varchar(50) , + `artist_name` varchar(50) , + `country` varchar(20) , + `f_id` bigINT , + `g_name` varchar(20) , + `rating` INT , + `languages` varchar(20) , + `releasedate` varchar(50) , + `resolution` bigINT NOT NULL + ); +COMMENT ON TABLE song IS 'song'; + +-- benchmark + create table s2_materialization ( id int AUTO_INCREMENT , @@ -469,13 +525,9 @@ CREATE TABLE s2_materialization_record PRIMARY KEY (`id`) ); - CREATE TABLE s2_sys_parameter ( id INT PRIMARY KEY AUTO_INCREMENT, admin varchar(500), parameters text null -); - - - +); \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DatasourceDetail.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java similarity index 52% rename from semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DatasourceDetail.java rename to semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java index a0a53e93c..1be10ae82 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DatasourceDetail.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.semantic.api.model.pojo; -import java.util.List; import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; @Data -public class DatasourceDetail { +public class ModelDetail { private String queryType; @@ -19,5 +21,11 @@ public class DatasourceDetail { private List measures; + public String getSqlQuery() { + if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) { + sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1); + } + return sqlQuery; + } } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceRelaReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceRelaReq.java deleted file mode 100644 index dad001e61..000000000 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceRelaReq.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.semantic.api.model.request; - -import javax.validation.constraints.NotNull; -import lombok.Data; - -@Data -public class DatasourceRelaReq { - - private Long id; - - @NotNull(message = "model id cat not be null") - private Long modelId; - - @NotNull(message = "datasource id cat not be null") - private Long datasourceFrom; - - @NotNull(message = "datasource id cat not be null") - private Long datasourceTo; - - @NotNull(message = "join key cat not be null") - private String joinKey; - -} \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java deleted file mode 100644 index 7a34edaf3..000000000 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatasourceReq.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.tencent.supersonic.semantic.api.model.request; - - -import com.google.common.collect.Lists; -import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; -import com.tencent.supersonic.semantic.api.model.pojo.Dim; -import com.tencent.supersonic.semantic.api.model.pojo.Identify; -import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; -import java.util.List; -import java.util.stream.Collectors; -import lombok.Data; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - - -@Data -public class DatasourceReq extends SchemaItem { - - private Long databaseId; - - private String queryType; - - private String sqlQuery; - - private String tableQuery; - - private Long modelId; - - private List identifiers; - - private List dimensions; - - private List measures; - - private String filterSql; - - - - public List getTimeDimension() { - if (CollectionUtils.isEmpty(dimensions)) { - return Lists.newArrayList(); - } - return dimensions.stream() - .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType())) - .collect(Collectors.toList()); - } - - public String getSqlQuery() { - if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) { - sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1); - } - return sqlQuery; - } - -} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java index 191cd0dde..44a8f8b92 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java @@ -18,9 +18,6 @@ public class DimensionReq extends SchemaItem { @NotNull(message = "expr can not be null") private String expr; - - private Long datasourceId; - //DATE ID CATEGORY private String semanticType = "CATEGORY"; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DomainReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DomainReq.java index f4aed6263..7d7394520 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DomainReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DomainReq.java @@ -1,14 +1,12 @@ package com.tencent.supersonic.semantic.api.model.request; - import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; +import lombok.Data; import java.util.ArrayList; import java.util.List; -import lombok.Data; - @Data public class DomainReq extends SchemaItem { @@ -24,4 +22,22 @@ public class DomainReq extends SchemaItem { private List admins = new ArrayList<>(); private List adminOrgs = new ArrayList<>(); + + public String getViewer() { + return String.join(",", viewers); + } + + public String getViewOrg() { + return String.join(",", viewOrgs); + } + + + public String getAdmin() { + return String.join(",", admins); + } + + public String getAdminOrg() { + return String.join(",", adminOrgs); + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java index 0d82d83a1..b3af891fe 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java @@ -1,24 +1,37 @@ package com.tencent.supersonic.semantic.api.model.request; +import com.google.common.collect.Lists; +import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; -import com.tencent.supersonic.semantic.api.model.pojo.Entity; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; @Data public class ModelReq extends SchemaItem { - private Long domainId = 0L; + private Long databaseId; - private Integer isOpen = 0; + private Long domainId; + + private String filterSql; + + private Integer isOpen; + + private List drillDownDimensions; private String alias; + private ModelDetail modelDetail; + private List viewers = new ArrayList<>(); private List viewOrgs = new ArrayList<>(); @@ -27,9 +40,16 @@ public class ModelReq extends SchemaItem { private List adminOrgs = new ArrayList<>(); - private Entity entity; - private List drillDownDimensions; + public List getTimeDimension() { + if (CollectionUtils.isEmpty(modelDetail.getDimensions())) { + return Lists.newArrayList(); + } + return modelDetail.getDimensions().stream() + .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType())) + .collect(Collectors.toList()); + } + public String getViewer() { return String.join(",", viewers); @@ -48,4 +68,5 @@ public class ModelReq extends SchemaItem { return String.join(",", adminOrgs); } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceRelaResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceRelaResp.java deleted file mode 100644 index 3cb409523..000000000 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceRelaResp.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.semantic.api.model.response; - -import java.util.Date; -import lombok.Data; - -@Data -public class DatasourceRelaResp { - - private Long id; - - private Long domainId; - - private Long datasourceFrom; - - private Long datasourceTo; - - private String joinKey; - - private Date createdAt; - - private String createdBy; - - private Date updatedAt; - - private String updatedBy; - -} \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java deleted file mode 100644 index 0ba3c2abf..000000000 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.tencent.supersonic.semantic.api.model.response; - -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; -import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; -import lombok.Data; - -@Data -public class DatasourceResp extends SchemaItem { - - private Long modelId; - - private Long databaseId; - - private DatasourceDetail datasourceDetail; - - private String depends; - - private String filterSql; - -} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimSchemaResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimSchemaResp.java index e4118d52a..3f9b09ddd 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimSchemaResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimSchemaResp.java @@ -3,10 +3,14 @@ package com.tencent.supersonic.semantic.api.model.response; import lombok.Data; import lombok.ToString; +import java.util.List; + @Data @ToString(callSuper = true) public class DimSchemaResp extends DimensionResp { private Long useCnt = 0L; + private List entityAlias; + } \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java index 408195217..251874510 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java @@ -22,13 +22,11 @@ public class DimensionResp extends SchemaItem { private String fullPath; - private Long datasourceId; + private String modelName; - private String datasourceName; + private String modelBizName; - private String datasourceBizName; - - private String datasourceFilterSql; + private String modelFilterSql; //DATE ID CATEGORY private String semanticType; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java index c05aed014..d644a9c41 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java @@ -46,8 +46,6 @@ public class MetricResp extends SchemaItem { private boolean hasAdminRes = false; - private String defaultAgg; - public void setTag(String tag) { if (StringUtils.isBlank(tag)) { tags = Lists.newArrayList(); @@ -80,4 +78,8 @@ public class MetricResp extends SchemaItem { } return typeParams.getMeasures(); } + + public String getDefaultAgg() { + return typeParams.getMeasures().get(0).getAgg(); + } } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java index 9d9574a5b..dddc5a194 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java @@ -1,43 +1,66 @@ package com.tencent.supersonic.semantic.api.model.response; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; -import com.tencent.supersonic.semantic.api.model.pojo.Entity; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; -import lombok.ToString; +import org.springframework.util.CollectionUtils; +import java.util.ArrayList; import java.util.List; @Data -@ToString public class ModelResp extends SchemaItem { private Long domainId; + private Long databaseId; + + private ModelDetail modelDetail; + + private String depends; + + private String filterSql; + + private List viewers = new ArrayList<>(); + + private List viewOrgs = new ArrayList<>(); + + private List admins = new ArrayList<>(); + + private List adminOrgs = new ArrayList<>(); + + private Integer isOpen; + + private List drillDownDimensions; + private String alias; - private List viewers; - - private List viewOrgs; - - private List admins; - - private List adminOrgs; - - private Integer isOpen = 0; + private String fullPath; private Integer dimensionCnt; private Integer metricCnt; - private Entity entity; - - private String fullPath; - - private List drillDownDimensions; public boolean openToAll() { return isOpen != null && isOpen == 1; } + public Identify getPrimaryIdentify() { + if (modelDetail == null) { + return null; + } + if (CollectionUtils.isEmpty(modelDetail.getIdentifiers())) { + return null; + } + for (Identify identify : modelDetail.getIdentifiers()) { + if (!CollectionUtils.isEmpty(identify.getEntityNames())) { + return identify; + } + } + return null; + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaRelaResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaRelaResp.java index 48e40e257..11fa48f40 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaRelaResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaRelaResp.java @@ -1,14 +1,15 @@ package com.tencent.supersonic.semantic.api.model.response; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class ModelSchemaRelaResp { private Long domainId; - private DatasourceResp datasource; + private ModelResp model; private List metrics; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaResp.java index 5d948a936..69485587f 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelSchemaResp.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.semantic.api.model.response; -import java.util.List; - +import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @AllArgsConstructor @NoArgsConstructor @@ -13,5 +15,20 @@ public class ModelSchemaResp extends ModelResp { private List metrics; private List dimensions; + private List modelRelas; + + public DimSchemaResp getPrimaryKey() { + Identify identify = getPrimaryIdentify(); + if (identify == null) { + return null; + } + for (DimSchemaResp dimension : dimensions) { + if (identify.getBizName().equals(dimension.getBizName())) { + dimension.setEntityAlias(identify.getEntityNames()); + return dimension; + } + } + return null; + } } \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DataModelYamlTpl.java similarity index 91% rename from semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java rename to semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DataModelYamlTpl.java index 61a922820..85243b9dc 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DatasourceYamlTpl.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/yaml/DataModelYamlTpl.java @@ -5,7 +5,7 @@ import lombok.Data; @Data -public class DatasourceYamlTpl { +public class DataModelYamlTpl { private String name; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java index 475f67b7c..978338491 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java @@ -1,17 +1,30 @@ package com.tencent.supersonic.semantic.api.query.request; -import java.util.Map; +import com.google.common.collect.Lists; import lombok.Data; import lombok.ToString; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; @Data @ToString public class QueryS2SQLReq { - private Long modelId; + private Set modelIds; private String sql; private Map variables; + public void setModelId(Long modelId) { + modelIds = new HashSet<>(); + modelIds.add(modelId); + } + + public List getModelIds() { + return Lists.newArrayList(modelIds); + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java index 2b6e1d097..d1819abf1 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java @@ -14,11 +14,6 @@ import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.semantic.api.query.pojo.Cache; import com.tencent.supersonic.semantic.api.query.pojo.Param; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -42,12 +37,20 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Data @Slf4j public class QueryStructReq { - private Long modelId; + private Set modelIds; private String modelName; private List groups = new ArrayList<>(); @@ -70,6 +73,19 @@ public class QueryStructReq { */ private String correctS2SQL; + public void setModelId(Long modelId) { + modelIds = new HashSet<>(); + modelIds.add(modelId); + } + + public List getModelIds() { + return Lists.newArrayList(modelIds); + } + + public Set getModelIdSet() { + return modelIds; + } + public List getGroups() { if (!CollectionUtils.isEmpty(this.groups)) { this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList()); @@ -107,7 +123,7 @@ public class QueryStructReq { public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); stringBuilder.append("\"modelId\":") - .append(modelId); + .append(modelIds); stringBuilder.append(",\"groups\":") .append(groups); stringBuilder.append(",\"aggregators\":") @@ -139,7 +155,7 @@ public class QueryStructReq { public String toString() { final StringBuilder sb = new StringBuilder("{"); sb.append("\"modelId\":") - .append(modelId); + .append(modelIds); sb.append(",\"groups\":") .append(groups); sb.append(",\"aggregators\":") @@ -179,7 +195,7 @@ public class QueryStructReq { QueryS2SQLReq result = new QueryS2SQLReq(); result.setSql(sql); - result.setModelId(queryStructReq.getModelId()); + result.setModelIds(queryStructReq.getModelIdSet()); result.setVariables(new HashMap<>()); return result; } diff --git a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java index 8a5c50a74..a054a7a4e 100644 --- a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java +++ b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java @@ -20,10 +20,10 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.materialization.domain.MaterializationConfService; import com.tencent.supersonic.semantic.materialization.domain.pojo.Materialization; @@ -32,8 +32,11 @@ import com.tencent.supersonic.semantic.materialization.domain.repository.Materia import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationRepository; import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationConverter; import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationZipperUtils; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.ModelService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -46,11 +49,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - @Slf4j @Service @@ -59,18 +57,16 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic private final MaterializationRepository materializationRepository; private final MaterializationElementRepository materializationElementRepository; private final ModelService modelService; - private final DatasourceService datasourceService; private final MaterializationZipperUtils materializationZipperUtils; private String typeAndIdSplit = "_"; public MaterializationConfServiceImpl(MaterializationRepository materializationRepository, MaterializationElementRepository materializationElementRepository, - ModelService modelService, DatasourceService datasourceService, + ModelService modelService, MaterializationZipperUtils materializationZipperUtils) { this.materializationRepository = materializationRepository; this.materializationElementRepository = materializationElementRepository; this.modelService = modelService; - this.datasourceService = datasourceService; this.materializationZipperUtils = materializationZipperUtils; } @@ -309,7 +305,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic ModelSchemaFilterReq modelFilter = new ModelSchemaFilterReq(); modelFilter.setModelIds(Arrays.asList(materializationResp.getModelId())); List modelSchemaRespList = modelService.fetchModelSchema(modelFilter); - List measureRespList = datasourceService.getMeasureListOfModel( + List measureRespList = modelService.getMeasureListOfModel( Lists.newArrayList(materializationResp.getModelId())); Map dimSchemaRespMap = new HashMap<>(); Map metricSchemaRespHashMap = new HashMap<>(); @@ -363,13 +359,11 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic @Override public List getMaterializationSourceResp( Long materializationId) { - MaterializationResp materializationResp = materializationRepository.getMaterialization( materializationId); Long modelId = materializationResp.getModelId(); - List modelDataSources = datasourceService.getDatasourceList(modelId); - Set dataSourceIds = new HashSet<>(); + Set modelIds = new HashSet<>(); Map> dataSourceDimensions = new HashMap<>(); Map> dataSourceMetrics = new HashMap<>(); MaterializationConfFilter materializationConfFilter = new MaterializationConfFilter(); @@ -380,7 +374,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic ModelSchemaFilterReq modelSchemaFilterReq = new ModelSchemaFilterReq(); modelSchemaFilterReq.setModelIds(Arrays.asList(modelId)); List modelSchemaRespList = modelService.fetchModelSchema(modelSchemaFilterReq); - List measureRespList = datasourceService.getMeasureListOfModel(Lists.newArrayList(modelId)); + List measureRespList = modelService.getMeasureListOfModel(Lists.newArrayList(modelId)); Set dimensionIds = new HashSet<>(); Set metricIds = new HashSet<>(); materializationElementRespList.stream().forEach(e -> { @@ -393,11 +387,11 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic }); modelSchemaRespList.stream().forEach(m -> { m.getDimensions().stream().filter(mm -> dimensionIds.contains(mm.getId())).forEach(mm -> { - if (!dataSourceDimensions.containsKey(mm.getDatasourceId())) { - dataSourceDimensions.put(mm.getDatasourceId(), new HashMap<>()); + if (!dataSourceDimensions.containsKey(mm.getModelId())) { + dataSourceDimensions.put(mm.getModelId(), new HashMap<>()); } - dataSourceDimensions.get(mm.getDatasourceId()).put(mm.getId(), mm.getBizName()); - dataSourceIds.add(mm.getDatasourceId()); + dataSourceDimensions.get(mm.getModelId()).put(mm.getId(), mm.getBizName()); + modelIds.add(mm.getModelId()); }); m.getMetrics().stream().filter(mm -> metricIds.contains(mm.getId())).forEach(mm -> { Long sourceId = 0L; @@ -412,27 +406,29 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic dataSourceMetrics.put(sourceId, new HashMap<>()); } dataSourceMetrics.get(sourceId).put(mm.getId(), mm.getBizName()); - dataSourceIds.add(sourceId); + modelIds.add(sourceId); } }); }); } List materializationSourceResps = new ArrayList<>(); - for (Long sourceId : dataSourceIds) { - Optional datasourceResp = modelDataSources.stream().filter(d -> d.getId().equals(sourceId)) - .findFirst(); - if (datasourceResp.isPresent()) { + for (Long sourceId : modelIds) { + //todo + // Optional modelResp = modelSchemaRespList.stream().filter(d -> d.getId().equals(sourceId)) + // .findFirst(); + Optional modelResp = Optional.empty(); + if (modelResp.isPresent()) { MaterializationSourceResp materializationSourceResp = MaterializationSourceResp.builder() .dataSourceId(sourceId) .materializationId(materializationId) .modelId(modelId) - .depends(datasourceResp.get().getDepends()) + .depends(modelResp.get().getDepends()) .materializedType(materializationResp.getMaterializedType()) .entities(materializationResp.getEntities()) .dateInfo(materializationResp.getDateInfo()) .updateCycle(materializationResp.getUpdateCycle()) .build(); - setDataSourceDb(datasourceResp.get(), materializationSourceResp); + setDataSourceDb(modelResp.get(), materializationSourceResp); materializationSourceResp.setMetrics( dataSourceMetrics.containsKey(sourceId) ? dataSourceMetrics.get(sourceId) : new HashMap<>()); @@ -456,16 +452,16 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic return 0L; } - private void setDataSourceDb(DatasourceResp datasourceResp, MaterializationSourceResp materializationSourceResp) { - if (Objects.nonNull(datasourceResp.getDatasourceDetail())) { + private void setDataSourceDb(ModelResp datasourceResp, MaterializationSourceResp materializationSourceResp) { + if (Objects.nonNull(datasourceResp.getModelDetail())) { String dbTable = ""; - if (Objects.nonNull(datasourceResp.getDatasourceDetail().getTableQuery()) - && !datasourceResp.getDatasourceDetail().getTableQuery().isEmpty()) { - dbTable = datasourceResp.getDatasourceDetail().getTableQuery(); + if (Objects.nonNull(datasourceResp.getModelDetail().getTableQuery()) + && !datasourceResp.getModelDetail().getTableQuery().isEmpty()) { + dbTable = datasourceResp.getModelDetail().getTableQuery(); } - if (Objects.nonNull(datasourceResp.getDatasourceDetail().getSqlQuery()) - && !datasourceResp.getDatasourceDetail().getSqlQuery().isEmpty()) { - dbTable = SqlParserSelectHelper.getDbTableName(datasourceResp.getDatasourceDetail().getSqlQuery()); + if (Objects.nonNull(datasourceResp.getModelDetail().getSqlQuery()) + && !datasourceResp.getModelDetail().getSqlQuery().isEmpty()) { + dbTable = SqlParserSelectHelper.getDbTableName(datasourceResp.getModelDetail().getSqlQuery()); } if (!dbTable.isEmpty()) { String[] db = dbTable.split("\\."); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java index d1f733883..5c989efe6 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java @@ -1,35 +1,30 @@ package com.tencent.supersonic.semantic.model.application; -import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.ItemDateResp; +import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; import com.tencent.supersonic.semantic.model.domain.ModelService; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; @Slf4j @Component @@ -38,18 +33,20 @@ public class CatalogImpl implements Catalog { private final DatabaseService databaseService; private final ModelService modelService; private final DimensionService dimensionService; - private final DatasourceService datasourceService; + private final ModelService datasourceService; private final MetricService metricService; + private final ModelRelaService modelRelaService; public CatalogImpl(DatabaseService databaseService, ModelService modelService, DimensionService dimensionService, - DatasourceService datasourceService, - MetricService metricService) { + ModelService datasourceService, + MetricService metricService, ModelRelaService modelRelaService) { this.databaseService = databaseService; this.modelService = modelService; this.dimensionService = dimensionService; this.datasourceService = datasourceService; this.metricService = metricService; + this.modelRelaService = modelRelaService; } public DatabaseResp getDatabase(Long id) { @@ -62,7 +59,7 @@ public class CatalogImpl implements Catalog { @Override public String getModelFullPath(Long modelId) { - ModelResp modelResp = modelService.getModelMap().get(modelId); + ModelResp modelResp = modelService.getModel(modelId); if (modelResp != null) { return modelResp.getFullPath(); } @@ -70,8 +67,8 @@ public class CatalogImpl implements Catalog { } @Override - public Map getModelFullPath() { - return modelService.getModelFullPathMap(); + public String getModelFullPath(List modelIds) { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); } @Override @@ -80,27 +77,21 @@ public class CatalogImpl implements Catalog { } @Override - public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List datasourceYamlTplList, List metricYamlTplList) { - datasourceService.getModelYamlTplByModelIds(modelIds, dimensionYamlMap, datasourceYamlTplList, - metricYamlTplList); + public List getModelRela(List modelIds) { + return modelRelaService.getModelRela(modelIds); } + @Override - public List getDimensions(Long modelId) { - MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId)); + public List getDimensions(List modelIds) { + MetaFilter metricFilter = new MetaFilter(modelIds); metricFilter.setStatus(StatusEnum.ONLINE.getCode()); return dimensionService.getDimensions(metricFilter); } @Override - public List getDatasourceList(Long modelId) { - return datasourceService.getDatasourceList(modelId); - } - - @Override - public List getMetrics(Long modelId) { - MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId)); + public List getMetrics(List modelIds) { + MetaFilter metricFilter = new MetaFilter(modelIds); return metricService.getMetrics(metricFilter); } @@ -110,33 +101,11 @@ public class CatalogImpl implements Catalog { } @Override - public String getAgg(List metricResps, List measureRespList, String metricBizName) { - try { - if (!CollectionUtils.isEmpty(metricResps)) { - Optional metric = metricResps.stream() - .filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst(); - if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty( - metric.get().getTypeParams().getMeasures())) { - if (!CollectionUtils.isEmpty(measureRespList)) { - String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName(); - Optional measure = measureRespList.stream() - .filter(Objects::nonNull) - .filter(m -> { - if (StringUtils.isNotEmpty(m.getBizName())) { - return m.getBizName().equalsIgnoreCase(measureName); - } - return false; - }) - .findFirst(); - if (measure.isPresent()) { - return measure.get().getAgg(); - } - } - } - } - } catch (Exception e) { - log.error("getAgg:", e); - } - return ""; + public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, + List dataModelYamlTplList, List metricYamlTplList, + Map modelIdName) { + datasourceService.getModelYamlTplByModelIds(modelIds, dimensionYamlMap, dataModelYamlTplList, + metricYamlTplList, modelIdName); } + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java index 5c4fd9edb..35a3b9b71 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatabaseServiceImpl.java @@ -3,25 +3,27 @@ package com.tencent.supersonic.semantic.model.application; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.request.DatabaseReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; +import com.tencent.supersonic.semantic.model.domain.DatabaseService; +import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor; import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptorFactory; import com.tencent.supersonic.semantic.model.domain.dataobject.DatabaseDO; +import com.tencent.supersonic.semantic.model.domain.pojo.Database; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import com.tencent.supersonic.semantic.model.domain.repository.DatabaseRepository; import com.tencent.supersonic.semantic.model.domain.utils.DatabaseConverter; import com.tencent.supersonic.semantic.model.domain.utils.JdbcDataSourceUtils; import com.tencent.supersonic.semantic.model.domain.utils.SqlUtils; -import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.pojo.Database; -import java.util.List; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.stream.Collectors; + @Slf4j @Service @@ -29,11 +31,11 @@ public class DatabaseServiceImpl implements DatabaseService { private final SqlUtils sqlUtils; private DatabaseRepository databaseRepository; - private DatasourceService datasourceService; + private ModelService datasourceService; public DatabaseServiceImpl(DatabaseRepository databaseRepository, SqlUtils sqlUtils, - @Lazy DatasourceService datasourceService) { + @Lazy ModelService datasourceService) { this.databaseRepository = databaseRepository; this.sqlUtils = sqlUtils; this.datasourceService = datasourceService; @@ -88,11 +90,13 @@ public class DatabaseServiceImpl implements DatabaseService { @Override public void deleteDatabase(Long databaseId) { - List datasourceResps = datasourceService.getDatasourceByDatabase(databaseId); - if (!CollectionUtils.isEmpty(datasourceResps)) { - List datasourceNames = datasourceResps.stream() - .map(DatasourceResp::getName).collect(Collectors.toList()); - String message = String.format("该数据库被数据源%s使用，无法删除", datasourceNames); + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setDatabaseId(databaseId); + List modelResps = datasourceService.getModelList(modelFilter); + if (!CollectionUtils.isEmpty(modelResps)) { + List datasourceNames = modelResps.stream() + .map(ModelResp::getName).collect(Collectors.toList()); + String message = String.format("该数据库被模型%s使用，无法删除", datasourceNames); throw new RuntimeException(message); } databaseRepository.deleteDatabase(databaseId); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java deleted file mode 100644 index 6a5f7dc49..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java +++ /dev/null @@ -1,355 +0,0 @@ -package com.tencent.supersonic.semantic.model.application; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; -import com.tencent.supersonic.semantic.api.model.pojo.Dim; -import com.tencent.supersonic.semantic.api.model.pojo.Identify; -import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; -import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; -import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; -import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; -import com.tencent.supersonic.semantic.api.model.request.DateInfoReq; -import com.tencent.supersonic.semantic.api.model.request.DimensionReq; -import com.tencent.supersonic.semantic.api.model.request.MetricReq; -import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; -import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.MetricService; -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.DateInfoDO; -import com.tencent.supersonic.semantic.model.domain.manager.DatasourceYamlManager; -import com.tencent.supersonic.semantic.model.domain.manager.DimensionYamlManager; -import com.tencent.supersonic.semantic.model.domain.manager.MetricYamlManager; -import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; -import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository; -import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; -import com.tencent.supersonic.semantic.model.domain.utils.DatasourceConverter; -import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter; -import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.BeanUtils; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; - - -@Service -@Slf4j -public class DatasourceServiceImpl implements DatasourceService { - - private DatasourceRepository datasourceRepository; - - private DatabaseService databaseService; - - private DimensionService dimensionService; - - private MetricService metricService; - - private DateInfoRepository dateInfoRepository; - - public DatasourceServiceImpl(DatasourceRepository datasourceRepository, - DatabaseService databaseService, - @Lazy DimensionService dimensionService, - @Lazy MetricService metricService, - DateInfoRepository dateInfoRepository) { - this.datasourceRepository = datasourceRepository; - this.databaseService = databaseService; - this.dimensionService = dimensionService; - this.metricService = metricService; - this.dateInfoRepository = dateInfoRepository; - } - - @Override - @Transactional - public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception { - checkName(datasourceReq); - checkExist(datasourceReq); - DatasourceDO datasourceDO = DatasourceConverter.convert(datasourceReq, user); - datasourceRepository.createDatasource(datasourceDO); - batchCreateDimension(datasourceDO, user); - batchCreateMetric(datasourceDO, user); - return DatasourceConverter.convert(datasourceDO); - } - - @Override - @Transactional - public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception { - checkName(datasourceReq); - DatasourceDO datasourceDO = datasourceRepository.getDatasourceById(datasourceReq.getId()); - DatasourceConverter.convert(datasourceDO, datasourceReq, user); - datasourceRepository.updateDatasource(datasourceDO); - batchCreateDimension(datasourceDO, user); - batchCreateMetric(datasourceDO, user); - return DatasourceConverter.convert(datasourceDO); - } - - @Override - public List getMeasureListOfModel(List modelIds) { - if (CollectionUtils.isEmpty(modelIds)) { - return Lists.newArrayList(); - } - List datasourceResps = getDatasourceList().stream().filter(datasourceResp -> - modelIds.contains(datasourceResp.getModelId())).collect(Collectors.toList()); - List measureResps = Lists.newArrayList(); - if (!CollectionUtils.isEmpty(datasourceResps)) { - for (DatasourceResp datasourceDesc : datasourceResps) { - DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail(); - List measures = datasourceDetail.getMeasures(); - if (!CollectionUtils.isEmpty(measures)) { - measureResps.addAll( - measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc)) - .collect(Collectors.toList())); - } - } - } - return measureResps; - } - - private void batchCreateDimension(DatasourceDO datasourceDO, User user) throws Exception { - List dimensionReqs = DatasourceConverter.convertDimensionList(datasourceDO); - dimensionService.createDimensionBatch(dimensionReqs, user); - } - - private void batchCreateMetric(DatasourceDO datasourceDO, User user) throws Exception { - List exprMetricReqs = DatasourceConverter.convertMetricList(datasourceDO); - metricService.createMetricBatch(exprMetricReqs, user); - } - - private Optional getDatasource(Long modelId, String bizName) { - List datasourceResps = getDatasourceList(modelId); - if (CollectionUtils.isEmpty(datasourceResps)) { - return Optional.empty(); - } - for (DatasourceResp datasourceDesc : datasourceResps) { - if (datasourceDesc.getBizName().equals(bizName)) { - return Optional.of(datasourceDesc); - } - } - return Optional.empty(); - } - - private void checkName(DatasourceReq datasourceReq) { - if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) { - String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName()); - throw new InvalidArgumentException(message); - } - List dims = datasourceReq.getDimensions(); - List measures = datasourceReq.getMeasures(); - List timeDims = datasourceReq.getTimeDimension(); - List identifies = datasourceReq.getIdentifiers(); - if (CollectionUtils.isEmpty(dims)) { - throw new InvalidArgumentException("缺少维度信息"); - } - if (CollectionUtils.isEmpty(identifies)) { - throw new InvalidArgumentException("缺少主键信息"); - } - if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) { - throw new InvalidArgumentException("有度量时, 不可缺少时间维度"); - } - for (Measure measure : measures) { - if (StringUtils.isNotBlank(measure.getName()) - && NameCheckUtils.containsSpecialCharacters(measure.getName())) { - String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName()); - throw new InvalidArgumentException(message); - } - } - for (Dim dim : dims) { - if (StringUtils.isNotBlank(dim.getName()) - && NameCheckUtils.containsSpecialCharacters(dim.getName())) { - String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName()); - throw new InvalidArgumentException(message); - } - } - for (Identify identify : identifies) { - if (StringUtils.isNotBlank(identify.getName()) - && NameCheckUtils.containsSpecialCharacters(identify.getName())) { - String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName()); - throw new InvalidArgumentException(message); - } - } - } - - private void checkExist(DatasourceReq datasourceReq) { - Optional datasourceRespOptional = getDatasource(datasourceReq.getModelId(), - datasourceReq.getBizName()); - if (datasourceRespOptional.isPresent()) { - throw new InvalidArgumentException("已存在相同名字的数据源:" + datasourceReq.getBizName()); - } - } - - @Override - public List getDatasourceList(Long modelId) { - return DatasourceConverter.convertList(datasourceRepository.getDatasourceList(modelId)); - } - - @Override - public List getDatasourceList() { - return DatasourceConverter.convertList(datasourceRepository.getDatasourceList()); - } - - @Override - public List getDatasourceByDatabase(Long databaseId) { - return DatasourceConverter.convertList(datasourceRepository.getDatasourceByDatabase(databaseId)); - } - - @Override - public List getDatasourceListNoMeasurePrefix(Long modelId) { - List datasourceResps = getDatasourceList(modelId); - for (DatasourceResp datasourceResp : datasourceResps) { - if (!CollectionUtils.isEmpty(datasourceResp.getDatasourceDetail().getMeasures())) { - for (Measure measure : datasourceResp.getDatasourceDetail().getMeasures()) { - measure.setBizName(Optional.ofNullable(measure.getBizName()).orElse("") - .replace(getDatasourcePrefix(datasourceResp.getBizName()), "")); - } - } - } - return datasourceResps; - } - - private String getDatasourcePrefix(String datasourceBizName) { - return String.format("%s_", datasourceBizName); - } - - - @Override - public Map getDatasourceMap() { - Map map = new HashMap<>(); - List datasourceResps = getDatasourceList(); - if (CollectionUtils.isEmpty(datasourceResps)) { - return map; - } - return datasourceResps.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1)); - } - - - @Override - public void deleteDatasource(Long id, User user) { - DatasourceDO datasourceDO = datasourceRepository.getDatasourceById(id); - if (datasourceDO == null) { - return; - } - checkDelete(id); - datasourceDO.setStatus(StatusEnum.DELETED.getCode()); - datasourceDO.setUpdatedAt(new Date()); - datasourceDO.setUpdatedBy(user.getName()); - datasourceRepository.updateDatasource(datasourceDO); - } - - private void checkDelete(Long datasourceId) { - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setDatasourceId(datasourceId); - List metricResps = metricService.getMetrics(metaFilter); - List dimensionResps = dimensionService.getDimensions(metaFilter); - if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) { - throw new RuntimeException("存在基于该数据源创建的指标和维度, 暂不能删除, 请确认"); - } - } - - @Override - public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) { - List itemDates = new ArrayList<>(); - List dimensions = dateInfoRepository.getDateInfos(dimension); - List metrics = dateInfoRepository.getDateInfos(metric); - - log.info("getDateDate, dimension:{}, dimensions dateInfo:{}", dimension, dimensions); - log.info("getDateDate, metric:{}, metrics dateInfo:{}", metric, metrics); - itemDates.addAll(convert(dimensions)); - itemDates.addAll(convert(metrics)); - - ItemDateResp itemDateDescriptor = calculateDateInternal(itemDates); - log.info("itemDateDescriptor:{}", itemDateDescriptor); - - return itemDateDescriptor; - } - - private List convert(List dateInfoDOList) { - List dateInfoCommendList = new ArrayList<>(); - dateInfoDOList.stream().forEach(dateInfoDO -> { - DateInfoReq dateInfoCommend = new DateInfoReq(); - BeanUtils.copyProperties(dateInfoDO, dateInfoCommend); - dateInfoCommend.setUnavailableDateList(JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class)); - dateInfoCommendList.add(dateInfoCommend); - }); - return dateInfoCommendList; - } - - private ItemDateResp calculateDateInternal(List itemDates) { - if (CollectionUtils.isEmpty(itemDates)) { - log.warn("itemDates is empty!"); - return null; - } - String dateFormat = itemDates.get(0).getDateFormat(); - String startDate = itemDates.get(0).getStartDate(); - String endDate = itemDates.get(0).getEndDate(); - String datePeriod = itemDates.get(0).getDatePeriod(); - List unavailableDateList = itemDates.get(0).getUnavailableDateList(); - for (DateInfoReq item : itemDates) { - String startDate1 = item.getStartDate(); - String endDate1 = item.getEndDate(); - List unavailableDateList1 = item.getUnavailableDateList(); - if (Strings.isNotEmpty(startDate1) && startDate1.compareTo(startDate) > 0) { - startDate = startDate1; - } - if (Strings.isNotEmpty(endDate1) && endDate1.compareTo(endDate) < 0) { - endDate = endDate1; - } - if (!CollectionUtils.isEmpty(unavailableDateList1)) { - unavailableDateList.addAll(unavailableDateList1); - } - } - - return new ItemDateResp(dateFormat, startDate, endDate, datePeriod, unavailableDateList); - - } - - @Override - public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List datasourceYamlTplList, List metricYamlTplList) { - for (Long modelId : modelIds) { - List datasourceResps = getDatasourceList(modelId); - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); - List metricResps = metricService.getMetrics(metaFilter); - metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(MetricConverter.metricInfo2Metric(metricResps))); - Long databaseId = datasourceResps.iterator().next().getDatabaseId(); - DatabaseResp databaseResp = databaseService.getDatabase(databaseId); - List dimensionResps = dimensionService.getDimensions(metaFilter); - for (DatasourceResp datasourceResp : datasourceResps) { - datasourceYamlTplList.add(DatasourceYamlManager.convert2YamlObj( - DatasourceConverter.datasourceInfo2Datasource(datasourceResp), databaseResp)); - if (!dimensionYamlMap.containsKey(datasourceResp.getBizName())) { - dimensionYamlMap.put(datasourceResp.getBizName(), new ArrayList<>()); - } - List dimensionRespList = dimensionResps.stream() - .filter(d -> d.getDatasourceBizName().equalsIgnoreCase(datasourceResp.getBizName())) - .collect(Collectors.toList()); - dimensionYamlMap.get(datasourceResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml( - DimensionConverter.dimensionInfo2Dimension(dimensionRespList))); - } - } - - } -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 7e772bd44..f63f5fad3 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -7,37 +7,30 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.ChatGptHelper; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.model.domain.DatabaseService; +import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository; import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; -import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.stream.Collectors; import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -46,6 +39,13 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + @Service @Slf4j @@ -54,8 +54,6 @@ public class DimensionServiceImpl implements DimensionService { private DimensionRepository dimensionRepository; - private DatasourceService datasourceService; - private ModelService modelService; private ChatGptHelper chatGptHelper; @@ -68,12 +66,10 @@ public class DimensionServiceImpl implements DimensionService { public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService, - DatasourceService datasourceService, ChatGptHelper chatGptHelper, DatabaseService databaseService) { this.modelService = modelService; this.dimensionRepository = dimensionRepository; - this.datasourceService = datasourceService; this.chatGptHelper = chatGptHelper; this.databaseService = databaseService; } @@ -185,7 +181,7 @@ public class DimensionServiceImpl implements DimensionService { @Override public DimensionResp getDimension(Long id) { DimensionDO dimensionDO = dimensionRepository.getDimensionById(id); - return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), new HashMap<>()); + return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>()); } @Override @@ -198,7 +194,7 @@ public class DimensionServiceImpl implements DimensionService { .doSelectPageInfo(() -> queryDimension(dimensionFilter)); PageInfo pageInfo = new PageInfo<>(); BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo); - pageInfo.setList(convertList(dimensionDOPageInfo.getList(), datasourceService.getDatasourceMap())); + pageInfo.setList(convertList(dimensionDOPageInfo.getList(), modelService.getModelMap())); return pageInfo; } @@ -210,7 +206,7 @@ public class DimensionServiceImpl implements DimensionService { public List getDimensions(MetaFilter metaFilter) { DimensionFilter dimensionFilter = new DimensionFilter(); BeanUtils.copyProperties(metaFilter, dimensionFilter); - return convertList(dimensionRepository.getDimension(dimensionFilter), datasourceService.getDatasourceMap()); + return convertList(dimensionRepository.getDimension(dimensionFilter), modelService.getModelMap()); } private List getDimensions(Long modelId) { @@ -218,13 +214,11 @@ public class DimensionServiceImpl implements DimensionService { } private List convertList(List dimensionDOS, - Map datasourceRespMap) { + Map modelRespMap) { List dimensionResps = Lists.newArrayList(); - Map modelFullPathMap = modelService.getModelFullPathMap(); if (!CollectionUtils.isEmpty(dimensionDOS)) { dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelFullPathMap, - datasourceRespMap)) + .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelRespMap)) .collect(Collectors.toList()); } return dimensionResps; @@ -251,19 +245,10 @@ public class DimensionServiceImpl implements DimensionService { @Override public List mockDimensionValueAlias(DimensionReq dimensionReq, User user) { - - List datasourceList = datasourceService.getDatasourceList(); - List collect = datasourceList.stream().filter(datasourceResp -> - datasourceResp.getId().equals(dimensionReq.getDatasourceId())).collect(Collectors.toList()); - - if (collect.isEmpty()) { - return null; - } - DatasourceResp datasourceResp = collect.get(0); - DatasourceDetail datasourceDetail = datasourceResp.getDatasourceDetail(); - String sqlQuery = datasourceDetail.getSqlQuery(); - - DatabaseResp database = databaseService.getDatabase(datasourceResp.getDatabaseId()); + ModelResp modelResp = modelService.getModel(dimensionReq.getModelId()); + ModelDetail modelDetail = modelResp.getModelDetail(); + String sqlQuery = modelDetail.getSqlQuery(); + DatabaseResp database = databaseService.getDatabase(modelResp.getDatabaseId()); String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery + ") as ai_talk group by ai_talk." + dimensionReq.getBizName(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java index ebc629b8d..a82ad254d 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java @@ -1,39 +1,38 @@ package com.tencent.supersonic.semantic.model.application; -import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; -import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.semantic.api.model.request.DomainReq; import com.tencent.supersonic.semantic.api.model.request.DomainUpdateReq; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDO; -import com.tencent.supersonic.semantic.model.domain.pojo.Domain; import com.tencent.supersonic.semantic.model.domain.repository.DomainRepository; import com.tencent.supersonic.semantic.model.domain.utils.DomainConvert; -import java.util.List; -import java.util.Date; -import java.util.Set; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Map; -import java.util.HashMap; -import java.util.Queue; -import java.util.LinkedList; -import java.util.Objects; -import java.util.Comparator; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.assertj.core.util.Sets; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.Set; +import java.util.stream.Collectors; + @Service @Slf4j @@ -54,22 +53,16 @@ public class DomainServiceImpl implements DomainService { @Override public void createDomain(DomainReq domainReq, User user) { - log.info("[create domain] cmd : {}", JSONObject.toJSONString(domainReq)); - Domain domain = DomainConvert.convert(domainReq); - log.info("[create domain] object:{}", JSONObject.toJSONString(domainReq)); - saveDomain(domain, user); + DomainDO domainDO = DomainConvert.convert(domainReq, user); + domainDO.setStatus(StatusEnum.ONLINE.getCode()); + domainRepository.createDomain(domainDO); } @Override public void updateDomain(DomainUpdateReq domainUpdateReq, User user) { + domainUpdateReq.updatedBy(user.getName()); DomainDO domainDO = getDomainDO(domainUpdateReq.getId()); - domainDO.setUpdatedAt(new Date()); - domainDO.setUpdatedBy(user.getName()); BeanMapper.mapper(domainUpdateReq, domainDO); - domainDO.setAdmin(String.join(",", domainUpdateReq.getAdmins())); - domainDO.setAdminOrg(String.join(",", domainUpdateReq.getAdminOrgs())); - domainDO.setViewer(String.join(",", domainUpdateReq.getViewers())); - domainDO.setViewOrg(String.join(",", domainUpdateReq.getViewOrgs())); domainRepository.updateDomain(domainDO); } @@ -161,13 +154,6 @@ public class DomainServiceImpl implements DomainService { return getDomainFullPathMap(); } - //保存并获取自增ID - private void saveDomain(Domain domain, User user) { - DomainDO domainDO = DomainConvert.convert(domain, user); - domainRepository.createDomain(domainDO); - domain.setId(domainDO.getId()); - } - private List convertList(List domainDOS) { List domainDescs = Lists.newArrayList(); if (CollectionUtils.isEmpty(domainDOS)) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index 2a2805d12..661b260d6 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -6,8 +6,8 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; @@ -20,27 +20,17 @@ import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DomainService; +import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository; import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; -import com.tencent.supersonic.semantic.model.domain.MetricService; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -48,6 +38,13 @@ import org.springframework.beans.BeanUtils; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; @Service @Slf4j @@ -59,8 +56,6 @@ public class MetricServiceImpl implements MetricService { private DomainService domainService; - private DatasourceService datasourceService; - private ChatGptHelper chatGptHelper; private ApplicationEventPublisher eventPublisher; @@ -68,13 +63,11 @@ public class MetricServiceImpl implements MetricService { public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, DomainService domainService, - DatasourceService datasourceService, ChatGptHelper chatGptHelper, ApplicationEventPublisher eventPublisher) { this.domainService = domainService; this.metricRepository = metricRepository; this.modelService = modelService; - this.datasourceService = datasourceService; this.chatGptHelper = chatGptHelper; this.eventPublisher = eventPublisher; } @@ -198,21 +191,7 @@ public class MetricServiceImpl implements MetricService { public List getMetrics(MetaFilter metaFilter) { MetricFilter metricFilter = new MetricFilter(); BeanUtils.copyProperties(metaFilter, metricFilter); - List metricResps = convertList(queryMetric(metricFilter)); - if (metricFilter.getDatasourceId() != null) { - return filterByDatasource(metricFilter.getDatasourceId(), metricResps); - } - return metricResps; - } - - private List filterByDatasource(Long datasourceId, List metricResps) { - return metricResps.stream().filter(metricResp -> { - Set datasourceIdSet = metricResp.getTypeParams().getMeasures().stream() - .map(Measure::getDatasourceId) - .filter(Objects::nonNull).collect(Collectors.toSet()); - return !CollectionUtils.isEmpty(datasourceIdSet) - && datasourceIdSet.contains(datasourceId); - }).collect(Collectors.toList()); + return convertList(queryMetric(metricFilter)); } private void fillAdminRes(List metricResps, User user) { @@ -257,20 +236,7 @@ public class MetricServiceImpl implements MetricService { if (metricDO == null) { return null; } - MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, new HashMap<>()); - List measures = metricResp.getMeasures(); - if (CollectionUtils.isEmpty(measures)) { - return metricResp; - } - Map datasourceResps = datasourceService.getDatasourceList().stream() - .collect(Collectors.toMap(DatasourceResp::getId, datasourceResp -> datasourceResp)); - measures.forEach(measure -> { - DatasourceResp datasourceResp = datasourceResps.get(measure.getDatasourceId()); - if (datasourceResp != null) { - measure.setDatasourceName(datasourceResp.getName()); - } - }); - return metricResp; + return MetricConverter.convert2MetricResp(metricDO, new HashMap<>()); } @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelRelaServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelRelaServiceImpl.java new file mode 100644 index 000000000..fa3c3b8fa --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelRelaServiceImpl.java @@ -0,0 +1,73 @@ +package com.tencent.supersonic.semantic.model.application; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.JoinCondition; +import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; +import com.tencent.supersonic.semantic.model.domain.dataobject.ModelRelaDO; +import com.tencent.supersonic.semantic.model.infrastructure.mapper.ModelRelaDOMapper; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class ModelRelaServiceImpl + extends ServiceImpl implements ModelRelaService { + + @Override + public void save(ModelRela modelRela, User user) { + modelRela.createdBy(user.getName()); + ModelRelaDO modelRelaDO = convert(modelRela); + save(modelRelaDO); + } + + @Override + public void update(ModelRela modelRela, User user) { + modelRela.updatedBy(user.getName()); + ModelRelaDO modelRelaDO = convert(modelRela); + updateById(modelRelaDO); + } + + @Override + public List getModelRelaList(Long domainId) { + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda().eq(ModelRelaDO::getDomainId, domainId); + return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); + } + + @Override + public List getModelRela(List modelIds) { + QueryWrapper wrapper = new QueryWrapper<>(); + if (CollectionUtils.isEmpty(modelIds)) { + wrapper.lambda().in(ModelRelaDO::getFromModelId, modelIds).or() + .in(ModelRelaDO::getToModelId, modelIds); + } + return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); + } + + @Override + public void delete(Long id) { + removeById(id); + } + + private ModelRela convert(ModelRelaDO modelRelaDO) { + ModelRela modelRela = new ModelRela(); + BeanMapper.mapper(modelRelaDO, modelRela); + modelRela.setJoinConditions(JSONObject.parseArray(modelRelaDO.getJoinCondition(), JoinCondition.class)); + return modelRela; + } + + private ModelRelaDO convert(ModelRela modelRelaReq) { + ModelRelaDO modelRelaDO = new ModelRelaDO(); + BeanMapper.mapper(modelRelaReq, modelRelaDO); + modelRelaDO.setJoinCondition(JSONObject.toJSONString(modelRelaReq.getJoinConditions())); + return modelRelaDO; + } + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 17d177805..911ed7df7 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -3,17 +3,25 @@ package com.tencent.supersonic.semantic.model.application; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; -import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.ItemDateResp; +import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; +import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; +import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; +import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; +import com.tencent.supersonic.semantic.api.model.request.DateInfoReq; +import com.tencent.supersonic.semantic.api.model.request.DimensionReq; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; +import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; @@ -22,117 +30,251 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; -import com.tencent.supersonic.semantic.model.domain.Catalog; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.dataobject.DateInfoDO; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; +import com.tencent.supersonic.semantic.model.domain.manager.DatasourceYamlManager; +import com.tencent.supersonic.semantic.model.domain.manager.DimensionYamlManager; +import com.tencent.supersonic.semantic.model.domain.manager.MetricYamlManager; +import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; +import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; -import com.tencent.supersonic.semantic.model.domain.utils.ModelConvert; +import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter; +import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; +import com.tencent.supersonic.semantic.model.domain.utils.ModelConverter; +import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; -@Slf4j + @Service +@Slf4j public class ModelServiceImpl implements ModelService { - private final ModelRepository modelRepository; - private final MetricService metricService; - private final DimensionService dimensionService; - private final DatasourceService datasourceService; - private final DomainService domainService; - private final UserService userService; - private final DatabaseService databaseService; - private final Catalog catalog; - private ApplicationEventPublisher eventPublisher; + private ModelRepository modelRepository; - public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService, - @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService, - @Lazy DomainService domainService, UserService userService, - @Lazy DatabaseService databaseService, - @Lazy Catalog catalog, ApplicationEventPublisher eventPublisher) { + private DatabaseService databaseService; + + private DimensionService dimensionService; + + private MetricService metricService; + + private DomainService domainService; + + private ModelRelaService modelRelaService; + + private UserService userService; + + private DateInfoRepository dateInfoRepository; + + public ModelServiceImpl(ModelRepository modelRepository, + DatabaseService databaseService, + @Lazy DimensionService dimensionService, + @Lazy MetricService metricService, + ModelRelaService modelRelaService, + DomainService domainService, + UserService userService, + DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; - this.metricService = metricService; - this.dimensionService = dimensionService; - this.datasourceService = datasourceService; - this.domainService = domainService; - this.userService = userService; this.databaseService = databaseService; - this.catalog = catalog; - this.eventPublisher = eventPublisher; + this.dimensionService = dimensionService; + this.metricService = metricService; + this.domainService = domainService; + this.modelRelaService = modelRelaService; + this.userService = userService; + this.dateInfoRepository = dateInfoRepository; } @Override - public void createModel(ModelReq modelReq, User user) { - modelReq.createdBy(user.getName()); - ModelDO modelDO = ModelConvert.convert(modelReq); - modelRepository.createModel(modelDO); + @Transactional + public ModelResp createModel(ModelReq modelReq, User user) throws Exception { + checkName(modelReq); + ModelDO datasourceDO = ModelConverter.convert(modelReq, user); + modelRepository.createModel(datasourceDO); + batchCreateDimension(datasourceDO, user); + batchCreateMetric(datasourceDO, user); + return ModelConverter.convert(datasourceDO); } @Override - public void updateModel(ModelReq modelReq, User user) { - ModelDO modelDO = getModelDO(modelReq.getId()); - modelReq.updatedBy(user.getName()); - int oldStatus = modelDO.getStatus(); - BeanMapper.mapper(modelReq, modelDO); - if (modelReq.getEntity() != null) { - modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); - } - if (modelReq.getDrillDownDimensions() != null) { - modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions())); - } + @Transactional + public ModelResp updateModel(ModelReq modelReq, User user) throws Exception { + checkName(modelReq); + ModelDO modelDO = modelRepository.getModelById(modelReq.getId()); + ModelConverter.convert(modelDO, modelReq, user); modelRepository.updateModel(modelDO); - statusPublish(oldStatus, modelDO); + batchCreateDimension(modelDO, user); + batchCreateMetric(modelDO, user); + return ModelConverter.convert(modelDO); } - private void statusPublish(Integer oldStatus, ModelDO modelDO) { - if (oldStatus.equals(modelDO.getStatus())) { - return; - } - if (oldStatus.equals(StatusEnum.ONLINE.getCode()) - && modelDO.getStatus().equals(StatusEnum.OFFLINE.getCode())) { - publishEvent(EventType.DELETE, modelDO); - } else if (oldStatus.equals(StatusEnum.OFFLINE.getCode()) - && modelDO.getStatus().equals(StatusEnum.ONLINE.getCode())) { - publishEvent(EventType.ADD, modelDO); - } + @Override + public List getModelList(ModelFilter modelFilter) { + return ModelConverter.convertList(modelRepository.getModelList(modelFilter)); } - private void publishEvent(EventType eventType, ModelDO modelDO) { - eventPublisher.publishEvent( - new DataEvent(this, metricService.getDataItems(modelDO.getId()), - eventType)); - eventPublisher.publishEvent( - new DataEvent(this, dimensionService.getDataItems(modelDO.getId()), - eventType)); + @Override + public Map getModelMap() { + Map map = new HashMap<>(); + List modelResps = getModelList(new ModelFilter()); + if (CollectionUtils.isEmpty(modelResps)) { + return map; + } + return modelResps.stream().collect(Collectors.toMap(ModelResp::getId, a -> a, (k1, k2) -> k1)); } @Override public void deleteModel(Long id, User user) { + ModelDO datasourceDO = modelRepository.getModelById(id); + if (datasourceDO == null) { + return; + } checkDelete(id); - ModelDO modelDO = getModelDO(id); - modelDO.setStatus(StatusEnum.DELETED.getCode()); - modelDO.setUpdatedAt(new Date()); - modelDO.setUpdatedBy(user.getName()); - modelRepository.updateModel(modelDO); + datasourceDO.setStatus(StatusEnum.DELETED.getCode()); + datasourceDO.setUpdatedAt(new Date()); + datasourceDO.setUpdatedBy(user.getName()); + modelRepository.updateModel(datasourceDO); + } + + @Override + public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) { + List itemDates = new ArrayList<>(); + List dimensions = dateInfoRepository.getDateInfos(dimension); + List metrics = dateInfoRepository.getDateInfos(metric); + + log.info("getDateDate, dimension:{}, dimensions dateInfo:{}", dimension, dimensions); + log.info("getDateDate, metric:{}, metrics dateInfo:{}", metric, metrics); + itemDates.addAll(convert(dimensions)); + itemDates.addAll(convert(metrics)); + + ItemDateResp itemDateDescriptor = calculateDateInternal(itemDates); + log.info("itemDateDescriptor:{}", itemDateDescriptor); + + return itemDateDescriptor; + } + + @Override + public List getMeasureListOfModel(List modelIds) { + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setIds(modelIds); + List modelResps = getModelList(modelFilter); + return modelResps.stream().flatMap(modelResp -> modelResp.getModelDetail().getMeasures() + .stream().map(measure -> ModelConverter.convert(measure, modelResp))).collect(Collectors.toList()); + } + + private void batchCreateDimension(ModelDO datasourceDO, User user) throws Exception { + List dimensionReqs = ModelConverter.convertDimensionList(datasourceDO); + dimensionService.createDimensionBatch(dimensionReqs, user); + } + + private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception { + List exprMetricReqs = ModelConverter.convertMetricList(datasourceDO); + metricService.createMetricBatch(exprMetricReqs, user); + } + + private void checkName(ModelReq modelReq) { + if (NameCheckUtils.containsSpecialCharacters(modelReq.getName())) { + String message = String.format("模型名称[%s]包含特殊字符, 请修改", modelReq.getName()); + throw new InvalidArgumentException(message); + } + List dims = modelReq.getModelDetail().getDimensions(); + List measures = modelReq.getModelDetail().getMeasures(); + List timeDims = modelReq.getTimeDimension(); + List identifies = modelReq.getModelDetail().getIdentifiers(); + if (CollectionUtils.isEmpty(dims)) { + throw new InvalidArgumentException("缺少维度信息"); + } + if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) { + throw new InvalidArgumentException("有度量时, 不可缺少时间维度"); + } + for (Measure measure : measures) { + if (StringUtils.isNotBlank(measure.getName()) + && NameCheckUtils.containsSpecialCharacters(measure.getName())) { + String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName()); + throw new InvalidArgumentException(message); + } + } + for (Dim dim : dims) { + if (StringUtils.isNotBlank(dim.getName()) + && NameCheckUtils.containsSpecialCharacters(dim.getName())) { + String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName()); + throw new InvalidArgumentException(message); + } + } + for (Identify identify : identifies) { + if (StringUtils.isNotBlank(identify.getName()) + && NameCheckUtils.containsSpecialCharacters(identify.getName())) { + String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName()); + throw new InvalidArgumentException(message); + } + } + } + + private void checkDelete(Long modelId) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); + if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) { + throw new RuntimeException("存在基于该模型创建的指标和维度, 暂不能删除, 请确认"); + } + } + + private ItemDateResp calculateDateInternal(List itemDates) { + if (CollectionUtils.isEmpty(itemDates)) { + log.warn("itemDates is empty!"); + return null; + } + String dateFormat = itemDates.get(0).getDateFormat(); + String startDate = itemDates.get(0).getStartDate(); + String endDate = itemDates.get(0).getEndDate(); + String datePeriod = itemDates.get(0).getDatePeriod(); + List unavailableDateList = itemDates.get(0).getUnavailableDateList(); + for (DateInfoReq item : itemDates) { + String startDate1 = item.getStartDate(); + String endDate1 = item.getEndDate(); + List unavailableDateList1 = item.getUnavailableDateList(); + if (Strings.isNotEmpty(startDate1) && startDate1.compareTo(startDate) > 0) { + startDate = startDate1; + } + if (Strings.isNotEmpty(endDate1) && endDate1.compareTo(endDate) < 0) { + endDate = endDate1; + } + if (!CollectionUtils.isEmpty(unavailableDateList1)) { + unavailableDateList.addAll(unavailableDateList1); + } + } + + return new ItemDateResp(dateFormat, startDate, endDate, datePeriod, unavailableDateList); } @Override @@ -145,7 +287,8 @@ public class ModelServiceImpl implements ModelService { modelRespSet = modelRespSet.stream().filter(modelResp -> modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet()); } - return fillMetricInfo(new ArrayList<>(modelRespSet)); + return fillMetricInfo(new ArrayList<>(modelRespSet)).stream() + .sorted(Comparator.comparingLong(SchemaItem::getId)).collect(Collectors.toList()); } public List getModelRespAuthInheritDomain(User user, AuthType authType) { @@ -153,7 +296,7 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(domainResps)) { return Lists.newArrayList(); } - List allModelList = getModelList(); + List allModelList = getModelList(new ModelFilter()); Set domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet()); return allModelList.stream().filter(modelResp -> domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList()); @@ -161,7 +304,7 @@ public class ModelServiceImpl implements ModelService { @Override public List getModelAuthList(User user, AuthType authTypeEnum) { - List modelResps = getModelList(); + List modelResps = getModelList(new ModelFilter()); Set orgIds = userService.getUserAllOrgId(user.getName()); List modelWithAuth = Lists.newArrayList(); if (authTypeEnum.equals(AuthType.ADMIN)) { @@ -182,7 +325,9 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(domainIds)) { return Lists.newArrayList(); } - List modelResps = getModelList(); + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setDomainIds(domainIds); + List modelResps = getModelList(modelFilter); if (CollectionUtils.isEmpty(modelResps)) { return modelResps; } @@ -190,75 +335,15 @@ public class ModelServiceImpl implements ModelService { domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList()); } - - @Override - public List getModelList(List modelIds) { - return getModelList().stream() - .filter(modelDO -> modelIds.contains(modelDO.getId())) - .collect(Collectors.toList()); - } - - @Override - public List getModelList() { - return convertList(modelRepository.getModelList()); - } - @Override public ModelResp getModel(Long id) { + ModelDO modelDO = getModelDO(id); + if (modelDO == null) { + return null; + } Map domainRespMap = domainService.getDomainList().stream() .collect(Collectors.toMap(DomainResp::getId, d -> d)); - return ModelConvert.convert(getModelDO(id), domainRespMap); - } - - private void checkDelete(Long id) { - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(id)); - List metricResps = metricService.getMetrics(metaFilter); - List dimensionResps = dimensionService.getDimensions(metaFilter); - List datasourceResps = datasourceService.getDatasourceList(id); - if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(datasourceResps) - || !CollectionUtils.isEmpty(dimensionResps)) { - throw new RuntimeException("该模型下存在数据源、指标或者维度, 暂不能删除, 请确认"); - } - } - - private List convertList(List modelDOS) { - List modelResps = Lists.newArrayList(); - if (CollectionUtils.isEmpty(modelDOS)) { - return modelResps; - } - Map domainRespMap = domainService.getDomainList() - .stream().collect(Collectors.toMap(DomainResp::getId, d -> d)); - return modelDOS.stream() - .map(modelDO -> ModelConvert.convert(modelDO, domainRespMap)) - .collect(Collectors.toList()); - } - - private List fillMetricInfo(List modelResps) { - if (CollectionUtils.isEmpty(modelResps)) { - return modelResps; - } - Map> metricMap = metricService.getMetrics(new MetricFilter()).stream() - .collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionMap = dimensionService.getDimensions(new DimensionFilter()).stream() - .collect(Collectors.groupingBy(DimensionResp::getModelId)); - modelResps.forEach(modelResp -> { - modelResp.setDimensionCnt(dimensionMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); - modelResp.setMetricCnt(metricMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); - }); - return modelResps; - } - - - @Override - public Map getModelMap() { - return getModelList().stream().collect(Collectors.toMap(ModelResp::getId, a -> a, (k1, k2) -> k1)); - } - - @Override - public Map getModelFullPathMap() { - return getModelList().stream().filter(m -> m != null && m.getFullPath() != null) - .collect(Collectors.toMap(ModelResp::getId, - ModelResp::getFullPath, (k1, k2) -> k1)); + return ModelConverter.convert(modelDO, domainRespMap); } @Override @@ -282,12 +367,9 @@ public class ModelServiceImpl implements ModelService { return Lists.newArrayList(); } - protected ModelDO getModelDO(Long id) { - return modelRepository.getModelById(id); - } - - private ModelSchemaResp fetchSingleModelSchema(ModelResp modelResp) { - Long modelId = modelResp.getId(); + @Override + public ModelSchemaResp fetchSingleModelSchema(Long modelId) { + ModelResp modelResp = getModel(modelId); ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); BeanUtils.copyProperties(modelResp, modelSchemaResp); modelSchemaResp.setDimensions(generateDimSchema(modelId)); @@ -295,12 +377,6 @@ public class ModelServiceImpl implements ModelService { return modelSchemaResp; } - @Override - public ModelSchemaResp fetchSingleModelSchema(Long modelId) { - ModelResp model = getModel(modelId); - return fetchSingleModelSchema(model); - } - @Override public List fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq) { List modelSchemaRespList = new ArrayList<>(); @@ -314,23 +390,24 @@ public class ModelServiceImpl implements ModelService { .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); Map> dimensionRespsMap = dimensionService.getDimensions(metaFilter) .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); - Map> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds) - .stream().collect(Collectors.groupingBy(MeasureResp::getModelId)); + List modelRelas = modelRelaService.getModelRela(modelIds); for (Long modelId : modelIds) { ModelResp modelResp = getModelMap().get(modelId); if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) { continue; } - List measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList()); List metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList()); List metricSchemaResps = metricResps.stream().map(metricResp -> - convert(metricResp, metricResps, measureResps, modelResp)).collect(Collectors.toList()); + convert(metricResp, modelResp)).collect(Collectors.toList()); List dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()) .stream().map(this::convert).collect(Collectors.toList()); ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); BeanUtils.copyProperties(modelResp, modelSchemaResp); modelSchemaResp.setDimensions(dimensionResps); modelSchemaResp.setMetrics(metricSchemaResps); + modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela + -> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId)) + .collect(Collectors.toList())); modelSchemaRespList.add(modelSchemaResp); } return modelSchemaRespList; @@ -338,20 +415,58 @@ public class ModelServiceImpl implements ModelService { @Override public DatabaseResp getDatabaseByModelId(Long modelId) { - List datasourceResps = datasourceService.getDatasourceList(modelId); - if (!CollectionUtils.isEmpty(datasourceResps)) { - Long databaseId = datasourceResps.iterator().next().getDatabaseId(); + ModelResp modelResp = getModel(modelId); + if (modelResp != null) { + Long databaseId = modelResp.getDatabaseId(); return databaseService.getDatabase(databaseId); } return null; } + @Override + public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) { + if (CollectionUtils.isEmpty(metaBatchReq.getIds())) { + return; + } + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setIds(metaBatchReq.getIds()); + List modelDOS = modelRepository.getModelList(modelFilter); + if (CollectionUtils.isEmpty(modelDOS)) { + return; + } + modelDOS = modelDOS.stream() + .peek(modelDO -> { + modelDO.setStatus(metaBatchReq.getStatus()); + modelDO.setUpdatedAt(new Date()); + modelDO.setUpdatedBy(user.getName()); + }) + .collect(Collectors.toList()); + modelRepository.batchUpdate(modelDOS); + } + + private List fillMetricInfo(List modelResps) { + if (CollectionUtils.isEmpty(modelResps)) { + return modelResps; + } + Map> metricMap = metricService.getMetrics(new MetricFilter()).stream() + .collect(Collectors.groupingBy(MetricResp::getModelId)); + Map> dimensionMap = dimensionService.getDimensions(new DimensionFilter()).stream() + .collect(Collectors.groupingBy(DimensionResp::getModelId)); + modelResps.forEach(modelResp -> { + modelResp.setDimensionCnt(dimensionMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); + modelResp.setMetricCnt(metricMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); + }); + return modelResps; + } + + protected ModelDO getModelDO(Long id) { + return modelRepository.getModelById(id); + } + private List generateMetricSchema(Long modelId, ModelResp modelResp) { List metricSchemaDescList = new ArrayList<>(); List metricResps = metricService.getMetrics(new MetaFilter(Lists.newArrayList(modelId))); - List measureResps = datasourceService.getMeasureListOfModel(Lists.newArrayList(modelId)); - metricResps.stream().forEach(metricResp -> - metricSchemaDescList.add(convert(metricResp, metricResps, measureResps, modelResp))); + metricResps.forEach(metricResp -> metricSchemaDescList.add(convert(metricResp, modelResp))); return metricSchemaDescList; } @@ -367,8 +482,7 @@ public class ModelServiceImpl implements ModelService { return dimSchemaResp; } - private MetricSchemaResp convert(MetricResp metricResp, List metricResps, - List measureResps, ModelResp modelResp) { + private MetricSchemaResp convert(MetricResp metricResp, ModelResp modelResp) { MetricSchemaResp metricSchemaResp = new MetricSchemaResp(); BeanUtils.copyProperties(metricResp, metricSchemaResp); RelateDimension relateDimension = metricResp.getRelateDimension(); @@ -377,11 +491,20 @@ public class ModelServiceImpl implements ModelService { .drillDownDimensions(modelResp.getDrillDownDimensions()).build()); } metricSchemaResp.setUseCnt(0L); - String agg = catalog.getAgg(metricResps, measureResps, metricSchemaResp.getBizName()); - metricSchemaResp.setDefaultAgg(agg); return metricSchemaResp; } + private List convert(List dateInfoDOList) { + List dateInfoCommendList = new ArrayList<>(); + dateInfoDOList.forEach(dateInfoDO -> { + DateInfoReq dateInfoCommend = new DateInfoReq(); + BeanUtils.copyProperties(dateInfoDO, dateInfoCommend); + dateInfoCommend.setUnavailableDateList(JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class)); + dateInfoCommendList.add(dateInfoCommend); + }); + return dateInfoCommendList; + } + private List generateModelIdsReq(ModelSchemaFilterReq filter) { if (Objects.nonNull(filter) && !CollectionUtils.isEmpty(filter.getModelIds())) { return filter.getModelIds(); @@ -411,9 +534,10 @@ public class ModelServiceImpl implements ModelService { } public static boolean checkViewerPermission(Set orgIds, User user, ModelResp modelResp) { - List admins = modelResp.getAdmins(); + if (checkAdminPermission(orgIds, user, modelResp)) { + return true; + } List viewers = modelResp.getViewers(); - List adminOrgs = modelResp.getAdminOrgs(); List viewOrgs = modelResp.getViewOrgs(); if (user.isSuperAdmin()) { return true; @@ -422,17 +546,12 @@ public class ModelServiceImpl implements ModelService { return true; } String userName = user.getName(); - if (admins.contains(userName) || viewers.contains(userName) || modelResp.getCreatedBy().equals(userName)) { + if (viewers.contains(userName) || modelResp.getCreatedBy().equals(userName)) { return true; } - if (CollectionUtils.isEmpty(adminOrgs) && CollectionUtils.isEmpty(viewOrgs)) { + if (CollectionUtils.isEmpty(viewOrgs)) { return false; } - for (String orgId : orgIds) { - if (adminOrgs.contains(orgId)) { - return true; - } - } for (String orgId : orgIds) { if (viewOrgs.contains(orgId)) { return true; @@ -440,4 +559,43 @@ public class ModelServiceImpl implements ModelService { } return false; } + + + @Override + public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, + List dataModelYamlTplList, List metricYamlTplList, + Map modelIdName) { + for (Long modelId : modelIds) { + ModelResp modelResp = getModel(modelId); + modelIdName.put(modelId, modelResp.getBizName()); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); + List metricResps = metricService.getMetrics(metaFilter); + metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(MetricConverter.metricInfo2Metric(metricResps))); + Long databaseId = modelResp.getDatabaseId(); + DatabaseResp databaseResp = databaseService.getDatabase(databaseId); + List dimensionResps = dimensionService.getDimensions(metaFilter); + + dataModelYamlTplList.add(DatasourceYamlManager.convert2YamlObj( + datasourceInfo2Datasource(modelResp), databaseResp)); + if (!dimensionYamlMap.containsKey(modelResp.getBizName())) { + dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>()); + } + List dimensionRespList = dimensionResps.stream() + .filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName())) + .collect(Collectors.toList()); + dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml( + DimensionConverter.dimensionInfo2Dimension(dimensionRespList))); + } + } + + public static Datasource datasourceInfo2Datasource(ModelResp modelResp) { + Datasource datasource = new Datasource(); + BeanUtils.copyProperties(modelResp, datasource); + datasource.setDatasourceDetail(modelResp.getModelDetail()); + datasource.setModelId(modelResp.getId()); + datasource.setDatabaseId(modelResp.getDatabaseId()); + return datasource; + } + + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java index b97442a32..9952d53e4 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java @@ -1,63 +1,64 @@ package com.tencent.supersonic.semantic.model.application; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.request.ViewInfoReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.semantic.api.model.response.ModelSchemaRelaResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; -import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; -import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaRelaResp; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; +import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; import java.util.Date; import java.util.List; -import org.assertj.core.util.Lists; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; @Service public class ViewInfoServiceImpl { private ViewInfoRepository viewInfoRepository; - private DatasourceService datasourceService; + private ModelService modelService; private DimensionService dimensionService; private MetricService metricService; - public ViewInfoServiceImpl(ViewInfoRepository viewInfoRepository, DatasourceService datasourceService, + public ViewInfoServiceImpl(ViewInfoRepository viewInfoRepository, ModelService modelService, MetricService metricService, DimensionService dimensionService) { this.viewInfoRepository = viewInfoRepository; this.dimensionService = dimensionService; this.metricService = metricService; - this.datasourceService = datasourceService; + this.modelService = modelService; } public List getViewInfoList(Long modelId) { return viewInfoRepository.getViewInfoList(modelId); } - public List getDomainSchema(Long modelId) { + public List getDomainSchema(Long domainId) { List domainSchemaRelaResps = Lists.newArrayList(); - List datasourceResps = datasourceService.getDatasourceList(modelId); - for (DatasourceResp datasourceResp : datasourceResps) { + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setDomainIds(Lists.newArrayList(domainId)); + List modelResps = modelService.getModelList(modelFilter); + for (ModelResp modelResp : modelResps) { ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp(); - Long datasourceId = datasourceResp.getId(); MetaFilter metaFilter = new MetaFilter(); - metaFilter.setModelIds(Lists.newArrayList(modelId)); - metaFilter.setDatasourceId(datasourceId); + metaFilter.setModelIds(Lists.newArrayList(modelResp.getId())); List metricResps = metricService.getMetrics(metaFilter); List dimensionResps = dimensionService.getDimensions(metaFilter); - domainSchemaRelaResp.setDatasource(datasourceResp); + domainSchemaRelaResp.setModel(modelResp); domainSchemaRelaResp.setDimensions(dimensionResps); domainSchemaRelaResp.setMetrics(metricResps); - domainSchemaRelaResp.setDomainId(modelId); + domainSchemaRelaResp.setDomainId(domainId); domainSchemaRelaResps.add(domainSchemaRelaResp); } return domainSchemaRelaResps; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java index 275516cd6..9b77f04f5 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java @@ -1,15 +1,15 @@ package com.tencent.supersonic.semantic.model.domain; +import com.tencent.supersonic.common.pojo.ItemDateResp; +import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; + import java.util.List; import java.util.Map; import java.util.Set; @@ -20,24 +20,23 @@ public interface Catalog { DatabaseResp getDatabaseByModelId(Long modelId); - List getDatasourceList(Long modelId); - String getModelFullPath(Long modelId); - Map getModelFullPath(); + String getModelFullPath(List modelIds); DimensionResp getDimension(String bizName, Long modelId); - List getDimensions(Long modelId); + List getDimensions(List modelIds); - List getMetrics(Long modelId); + List getMetrics(List modelIds); + + List getModelRela(List modelIds); void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List datasourceYamlTplList, List metricYamlTplList); + List dataModelYamlTplList, List metricYamlTplList, + Map modelIdName); ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); - String getAgg(List metricResps, List measureRespList, String metricBizName); - } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java deleted file mode 100644 index 63c859cd8..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain; - - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; -import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; -import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public interface DatasourceService { - - DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception; - - DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception; - - List getDatasourceListNoMeasurePrefix(Long modelId); - - List getDatasourceList(); - - List getDatasourceList(Long modelId); - - List getDatasourceByDatabase(Long databaseId); - - Map getDatasourceMap(); - - void deleteDatasource(Long id, User user); - - ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); - - List getMeasureListOfModel(List modelIds); - - void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List datasourceYamlTplList, List metricYamlTplList); - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelRelaService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelRelaService.java new file mode 100644 index 000000000..a7ac31668 --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelRelaService.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.semantic.model.domain; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.ModelRela; + +import java.util.List; + +public interface ModelRelaService { + + + void save(ModelRela modelRela, User user); + + void update(ModelRela modelRela, User user); + + List getModelRelaList(Long domainId); + + List getModelRela(List modelIds); + + void delete(Long id); +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java index 2032f4b79..f77e80914 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java @@ -1,39 +1,48 @@ package com.tencent.supersonic.semantic.model.domain; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; +import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import java.util.List; import java.util.Map; +import java.util.Set; public interface ModelService { + ModelResp createModel(ModelReq datasourceReq, User user) throws Exception; + + ModelResp updateModel(ModelReq datasourceReq, User user) throws Exception; + + List getModelList(ModelFilter modelFilter); + + Map getModelMap(); + + void deleteModel(Long id, User user); + + ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); + + List getMeasureListOfModel(List modelIds); + List getModelListWithAuth(User user, Long domainId, AuthType authType); List getModelAuthList(User user, AuthType authTypeEnum); List getModelByDomainIds(List domainIds); - List getModelList(); - - List getModelList(List modelIds); - ModelResp getModel(Long id); - void updateModel(ModelReq modelReq, User user); - - void createModel(ModelReq modelReq, User user); - - void deleteModel(Long id, User user); - - Map getModelMap(); - - Map getModelFullPathMap(); - List getModelAdmin(Long id); ModelSchemaResp fetchSingleModelSchema(Long modelId); @@ -41,4 +50,11 @@ public interface ModelService { List fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq); DatabaseResp getDatabaseByModelId(Long modelId); + + void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); + + void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, + List dataModelYamlTplList, List metricYamlTplList, + Map modelIdName); + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java deleted file mode 100644 index 28bdca57d..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.dataobject; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import java.util.Date; - -@Data -@TableName("s2_datasource") -public class DatasourceDO { - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 模型ID - */ - private Long modelId; - - /** - * 数据源名称 - */ - private String name; - - /** - * 内部名称 - */ - private String bizName; - - /** - * 数据源描述 - */ - private String description; - - /** - * 数据库实例ID - */ - private Long databaseId; - - /** - * - */ - private Integer status; - - /** - * 创建时间 - */ - private Date createdAt; - - /** - * 创建人 - */ - private String createdBy; - - /** - * 更新时间 - */ - private Date updatedAt; - - /** - * 更新人 - */ - private String updatedBy; - - /** - * 数据源配置 - */ - private String datasourceDetail; - - /** - * 上游依赖标识 - */ - private String depends; - - private String filterSql; - -} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java deleted file mode 100644 index 0fcf4c1b1..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java +++ /dev/null @@ -1,1015 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class DatasourceDOExample { - /** - * s2_datasource - */ - protected String orderByClause; - - /** - * s2_datasource - */ - protected boolean distinct; - - /** - * s2_datasource - */ - protected List oredCriteria; - - /** - * s2_datasource - */ - protected Integer limitStart; - - /** - * s2_datasource - */ - protected Integer limitEnd; - - /** - * - * @mbg.generated - */ - public DatasourceDOExample() { - oredCriteria = new ArrayList(); - } - - /** - * - * @mbg.generated - */ - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - /** - * - * @mbg.generated - */ - public String getOrderByClause() { - return orderByClause; - } - - /** - * - * @mbg.generated - */ - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - /** - * - * @mbg.generated - */ - public boolean isDistinct() { - return distinct; - } - - /** - * - * @mbg.generated - */ - public List getOredCriteria() { - return oredCriteria; - } - - /** - * - * @mbg.generated - */ - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - /** - * - * @mbg.generated - */ - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - /** - * - * @mbg.generated - */ - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - /** - * - * @mbg.generated - */ - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - /** - * - * @mbg.generated - */ - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - /** - * - * @mbg.generated - */ - public void setLimitStart(Integer limitStart) { - this.limitStart=limitStart; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitStart() { - return limitStart; - } - - /** - * - * @mbg.generated - */ - public void setLimitEnd(Integer limitEnd) { - this.limitEnd=limitEnd; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitEnd() { - return limitEnd; - } - - /** - * s2_datasource null - */ - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andModelIdIsNull() { - addCriterion("model_id is null"); - return (Criteria) this; - } - - public Criteria andModelIdIsNotNull() { - addCriterion("model_id is not null"); - return (Criteria) this; - } - - public Criteria andModelIdEqualTo(Long value) { - addCriterion("model_id =", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotEqualTo(Long value) { - addCriterion("model_id <>", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThan(Long value) { - addCriterion("model_id >", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThanOrEqualTo(Long value) { - addCriterion("model_id >=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThan(Long value) { - addCriterion("model_id <", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThanOrEqualTo(Long value) { - addCriterion("model_id <=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdIn(List values) { - addCriterion("model_id in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotIn(List values) { - addCriterion("model_id not in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdBetween(Long value1, Long value2) { - addCriterion("model_id between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotBetween(Long value1, Long value2) { - addCriterion("model_id not between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andBizNameIsNull() { - addCriterion("biz_name is null"); - return (Criteria) this; - } - - public Criteria andBizNameIsNotNull() { - addCriterion("biz_name is not null"); - return (Criteria) this; - } - - public Criteria andBizNameEqualTo(String value) { - addCriterion("biz_name =", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotEqualTo(String value) { - addCriterion("biz_name <>", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThan(String value) { - addCriterion("biz_name >", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThanOrEqualTo(String value) { - addCriterion("biz_name >=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThan(String value) { - addCriterion("biz_name <", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThanOrEqualTo(String value) { - addCriterion("biz_name <=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLike(String value) { - addCriterion("biz_name like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotLike(String value) { - addCriterion("biz_name not like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameIn(List values) { - addCriterion("biz_name in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotIn(List values) { - addCriterion("biz_name not in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameBetween(String value1, String value2) { - addCriterion("biz_name between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotBetween(String value1, String value2) { - addCriterion("biz_name not between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNull() { - addCriterion("description is null"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNotNull() { - addCriterion("description is not null"); - return (Criteria) this; - } - - public Criteria andDescriptionEqualTo(String value) { - addCriterion("description =", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotEqualTo(String value) { - addCriterion("description <>", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThan(String value) { - addCriterion("description >", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThanOrEqualTo(String value) { - addCriterion("description >=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThan(String value) { - addCriterion("description <", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThanOrEqualTo(String value) { - addCriterion("description <=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLike(String value) { - addCriterion("description like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotLike(String value) { - addCriterion("description not like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionIn(List values) { - addCriterion("description in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotIn(List values) { - addCriterion("description not in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionBetween(String value1, String value2) { - addCriterion("description between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotBetween(String value1, String value2) { - addCriterion("description not between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDatabaseIdIsNull() { - addCriterion("database_id is null"); - return (Criteria) this; - } - - public Criteria andDatabaseIdIsNotNull() { - addCriterion("database_id is not null"); - return (Criteria) this; - } - - public Criteria andDatabaseIdEqualTo(Long value) { - addCriterion("database_id =", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdNotEqualTo(Long value) { - addCriterion("database_id <>", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdGreaterThan(Long value) { - addCriterion("database_id >", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdGreaterThanOrEqualTo(Long value) { - addCriterion("database_id >=", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdLessThan(Long value) { - addCriterion("database_id <", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdLessThanOrEqualTo(Long value) { - addCriterion("database_id <=", value, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdIn(List values) { - addCriterion("database_id in", values, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdNotIn(List values) { - addCriterion("database_id not in", values, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdBetween(Long value1, Long value2) { - addCriterion("database_id between", value1, value2, "databaseId"); - return (Criteria) this; - } - - public Criteria andDatabaseIdNotBetween(Long value1, Long value2) { - addCriterion("database_id not between", value1, value2, "databaseId"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(Integer value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(Integer value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(Integer value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(Integer value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(Integer value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(Integer value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(Integer value1, Integer value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(Integer value1, Integer value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNull() { - addCriterion("created_at is null"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNotNull() { - addCriterion("created_at is not null"); - return (Criteria) this; - } - - public Criteria andCreatedAtEqualTo(Date value) { - addCriterion("created_at =", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotEqualTo(Date value) { - addCriterion("created_at <>", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThan(Date value) { - addCriterion("created_at >", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("created_at >=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThan(Date value) { - addCriterion("created_at <", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThanOrEqualTo(Date value) { - addCriterion("created_at <=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtIn(List values) { - addCriterion("created_at in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotIn(List values) { - addCriterion("created_at not in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtBetween(Date value1, Date value2) { - addCriterion("created_at between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotBetween(Date value1, Date value2) { - addCriterion("created_at not between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNull() { - addCriterion("created_by is null"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNotNull() { - addCriterion("created_by is not null"); - return (Criteria) this; - } - - public Criteria andCreatedByEqualTo(String value) { - addCriterion("created_by =", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotEqualTo(String value) { - addCriterion("created_by <>", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThan(String value) { - addCriterion("created_by >", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThanOrEqualTo(String value) { - addCriterion("created_by >=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThan(String value) { - addCriterion("created_by <", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThanOrEqualTo(String value) { - addCriterion("created_by <=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLike(String value) { - addCriterion("created_by like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotLike(String value) { - addCriterion("created_by not like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByIn(List values) { - addCriterion("created_by in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotIn(List values) { - addCriterion("created_by not in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByBetween(String value1, String value2) { - addCriterion("created_by between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotBetween(String value1, String value2) { - addCriterion("created_by not between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNull() { - addCriterion("updated_at is null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNotNull() { - addCriterion("updated_at is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtEqualTo(Date value) { - addCriterion("updated_at =", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotEqualTo(Date value) { - addCriterion("updated_at <>", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThan(Date value) { - addCriterion("updated_at >", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("updated_at >=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThan(Date value) { - addCriterion("updated_at <", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThanOrEqualTo(Date value) { - addCriterion("updated_at <=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIn(List values) { - addCriterion("updated_at in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotIn(List values) { - addCriterion("updated_at not in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtBetween(Date value1, Date value2) { - addCriterion("updated_at between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotBetween(Date value1, Date value2) { - addCriterion("updated_at not between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNull() { - addCriterion("updated_by is null"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNotNull() { - addCriterion("updated_by is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedByEqualTo(String value) { - addCriterion("updated_by =", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotEqualTo(String value) { - addCriterion("updated_by <>", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThan(String value) { - addCriterion("updated_by >", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThanOrEqualTo(String value) { - addCriterion("updated_by >=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThan(String value) { - addCriterion("updated_by <", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThanOrEqualTo(String value) { - addCriterion("updated_by <=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLike(String value) { - addCriterion("updated_by like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotLike(String value) { - addCriterion("updated_by not like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByIn(List values) { - addCriterion("updated_by in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotIn(List values) { - addCriterion("updated_by not in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByBetween(String value1, String value2) { - addCriterion("updated_by between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotBetween(String value1, String value2) { - addCriterion("updated_by not between", value1, value2, "updatedBy"); - return (Criteria) this; - } - } - - /** - * s2_datasource - */ - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - /** - * s2_datasource null - */ - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDO.java index cfad208d8..d52e64ac1 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDO.java @@ -9,105 +9,44 @@ import java.util.Date; @Data @TableName("s2_dimension") public class DimensionDO { - /** - * 维度ID - */ + @TableId(type = IdType.AUTO) private Long id; - /** - * 主题域id - */ private Long modelId; - /** - * 所属数据源id - */ - private Long datasourceId; - - /** - * 维度名称 - */ private String name; - /** - * 字段名称 - */ private String bizName; - /** - * 描述 - */ private String description; - /** - * 维度状态,0正常,1下架,2删除 - */ private Integer status; - /** - * 敏感级别 - */ private Integer sensitiveLevel; - /** - * 维度类型 categorical,time - */ private String type; - /** - * 创建时间 - */ private Date createdAt; - /** - * 创建人 - */ private String createdBy; - /** - * 更新时间 - */ private Date updatedAt; - /** - * 更新人 - */ private String updatedBy; - /** - * 语义类型DATE, ID, CATEGORY - */ private String semanticType; - /** - * - */ private String alias; - /** - * default values of dimension when query - */ private String defaultValues; - /** - * - */ private String dimValueMaps; - /** - * 类型参数 - */ private String typeParams; - /** - * 表达式 - */ private String expr; - /** - * 数据类型 - */ private String dataType; private int isTag; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDOExample.java deleted file mode 100644 index 14d62dec9..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDOExample.java +++ /dev/null @@ -1,1410 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class DimensionDOExample { - /** - * s2_dimension - */ - protected String orderByClause; - - /** - * s2_dimension - */ - protected boolean distinct; - - /** - * s2_dimension - */ - protected List oredCriteria; - - /** - * s2_dimension - */ - protected Integer limitStart; - - /** - * s2_dimension - */ - protected Integer limitEnd; - - /** - * @mbg.generated - */ - public DimensionDOExample() { - oredCriteria = new ArrayList(); - } - - /** - * @mbg.generated - */ - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - /** - * @mbg.generated - */ - public String getOrderByClause() { - return orderByClause; - } - - /** - * @mbg.generated - */ - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - /** - * @mbg.generated - */ - public boolean isDistinct() { - return distinct; - } - - /** - * @mbg.generated - */ - public List getOredCriteria() { - return oredCriteria; - } - - /** - * @mbg.generated - */ - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - /** - * @mbg.generated - */ - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - /** - * @mbg.generated - */ - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - /** - * @mbg.generated - */ - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - /** - * @mbg.generated - */ - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - /** - * @mbg.generated - */ - public void setLimitStart(Integer limitStart) { - this.limitStart = limitStart; - } - - /** - * @mbg.generated - */ - public Integer getLimitStart() { - return limitStart; - } - - /** - * @mbg.generated - */ - public void setLimitEnd(Integer limitEnd) { - this.limitEnd = limitEnd; - } - - /** - * @mbg.generated - */ - public Integer getLimitEnd() { - return limitEnd; - } - - /** - * s2_dimension null - */ - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andModelIdIsNull() { - addCriterion("model_id is null"); - return (Criteria) this; - } - - public Criteria andModelIdIsNotNull() { - addCriterion("model_id is not null"); - return (Criteria) this; - } - - public Criteria andModelIdEqualTo(Long value) { - addCriterion("model_id =", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotEqualTo(Long value) { - addCriterion("model_id <>", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThan(Long value) { - addCriterion("model_id >", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThanOrEqualTo(Long value) { - addCriterion("model_id >=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThan(Long value) { - addCriterion("model_id <", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThanOrEqualTo(Long value) { - addCriterion("model_id <=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdIn(List values) { - addCriterion("model_id in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotIn(List values) { - addCriterion("model_id not in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdBetween(Long value1, Long value2) { - addCriterion("model_id between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotBetween(Long value1, Long value2) { - addCriterion("model_id not between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdIsNull() { - addCriterion("datasource_id is null"); - return (Criteria) this; - } - - public Criteria andDatasourceIdIsNotNull() { - addCriterion("datasource_id is not null"); - return (Criteria) this; - } - - public Criteria andDatasourceIdEqualTo(Long value) { - addCriterion("datasource_id =", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdNotEqualTo(Long value) { - addCriterion("datasource_id <>", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdGreaterThan(Long value) { - addCriterion("datasource_id >", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdGreaterThanOrEqualTo(Long value) { - addCriterion("datasource_id >=", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdLessThan(Long value) { - addCriterion("datasource_id <", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdLessThanOrEqualTo(Long value) { - addCriterion("datasource_id <=", value, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdIn(List values) { - addCriterion("datasource_id in", values, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdNotIn(List values) { - addCriterion("datasource_id not in", values, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdBetween(Long value1, Long value2) { - addCriterion("datasource_id between", value1, value2, "datasourceId"); - return (Criteria) this; - } - - public Criteria andDatasourceIdNotBetween(Long value1, Long value2) { - addCriterion("datasource_id not between", value1, value2, "datasourceId"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andBizNameIsNull() { - addCriterion("biz_name is null"); - return (Criteria) this; - } - - public Criteria andBizNameIsNotNull() { - addCriterion("biz_name is not null"); - return (Criteria) this; - } - - public Criteria andBizNameEqualTo(String value) { - addCriterion("biz_name =", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotEqualTo(String value) { - addCriterion("biz_name <>", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThan(String value) { - addCriterion("biz_name >", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThanOrEqualTo(String value) { - addCriterion("biz_name >=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThan(String value) { - addCriterion("biz_name <", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThanOrEqualTo(String value) { - addCriterion("biz_name <=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLike(String value) { - addCriterion("biz_name like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotLike(String value) { - addCriterion("biz_name not like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameIn(List values) { - addCriterion("biz_name in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotIn(List values) { - addCriterion("biz_name not in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameBetween(String value1, String value2) { - addCriterion("biz_name between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotBetween(String value1, String value2) { - addCriterion("biz_name not between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNull() { - addCriterion("description is null"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNotNull() { - addCriterion("description is not null"); - return (Criteria) this; - } - - public Criteria andDescriptionEqualTo(String value) { - addCriterion("description =", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotEqualTo(String value) { - addCriterion("description <>", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThan(String value) { - addCriterion("description >", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThanOrEqualTo(String value) { - addCriterion("description >=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThan(String value) { - addCriterion("description <", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThanOrEqualTo(String value) { - addCriterion("description <=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLike(String value) { - addCriterion("description like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotLike(String value) { - addCriterion("description not like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionIn(List values) { - addCriterion("description in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotIn(List values) { - addCriterion("description not in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionBetween(String value1, String value2) { - addCriterion("description between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotBetween(String value1, String value2) { - addCriterion("description not between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(Integer value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(Integer value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(Integer value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(Integer value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(Integer value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(Integer value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(Integer value1, Integer value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(Integer value1, Integer value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIsNull() { - addCriterion("sensitive_level is null"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIsNotNull() { - addCriterion("sensitive_level is not null"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelEqualTo(Integer value) { - addCriterion("sensitive_level =", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotEqualTo(Integer value) { - addCriterion("sensitive_level <>", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelGreaterThan(Integer value) { - addCriterion("sensitive_level >", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelGreaterThanOrEqualTo(Integer value) { - addCriterion("sensitive_level >=", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelLessThan(Integer value) { - addCriterion("sensitive_level <", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelLessThanOrEqualTo(Integer value) { - addCriterion("sensitive_level <=", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIn(List values) { - addCriterion("sensitive_level in", values, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotIn(List values) { - addCriterion("sensitive_level not in", values, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelBetween(Integer value1, Integer value2) { - addCriterion("sensitive_level between", value1, value2, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotBetween(Integer value1, Integer value2) { - addCriterion("sensitive_level not between", value1, value2, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNull() { - addCriterion("created_at is null"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNotNull() { - addCriterion("created_at is not null"); - return (Criteria) this; - } - - public Criteria andCreatedAtEqualTo(Date value) { - addCriterion("created_at =", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotEqualTo(Date value) { - addCriterion("created_at <>", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThan(Date value) { - addCriterion("created_at >", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("created_at >=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThan(Date value) { - addCriterion("created_at <", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThanOrEqualTo(Date value) { - addCriterion("created_at <=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtIn(List values) { - addCriterion("created_at in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotIn(List values) { - addCriterion("created_at not in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtBetween(Date value1, Date value2) { - addCriterion("created_at between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotBetween(Date value1, Date value2) { - addCriterion("created_at not between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNull() { - addCriterion("created_by is null"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNotNull() { - addCriterion("created_by is not null"); - return (Criteria) this; - } - - public Criteria andCreatedByEqualTo(String value) { - addCriterion("created_by =", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotEqualTo(String value) { - addCriterion("created_by <>", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThan(String value) { - addCriterion("created_by >", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThanOrEqualTo(String value) { - addCriterion("created_by >=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThan(String value) { - addCriterion("created_by <", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThanOrEqualTo(String value) { - addCriterion("created_by <=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLike(String value) { - addCriterion("created_by like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotLike(String value) { - addCriterion("created_by not like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByIn(List values) { - addCriterion("created_by in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotIn(List values) { - addCriterion("created_by not in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByBetween(String value1, String value2) { - addCriterion("created_by between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotBetween(String value1, String value2) { - addCriterion("created_by not between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNull() { - addCriterion("updated_at is null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNotNull() { - addCriterion("updated_at is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtEqualTo(Date value) { - addCriterion("updated_at =", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotEqualTo(Date value) { - addCriterion("updated_at <>", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThan(Date value) { - addCriterion("updated_at >", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("updated_at >=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThan(Date value) { - addCriterion("updated_at <", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThanOrEqualTo(Date value) { - addCriterion("updated_at <=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIn(List values) { - addCriterion("updated_at in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotIn(List values) { - addCriterion("updated_at not in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtBetween(Date value1, Date value2) { - addCriterion("updated_at between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotBetween(Date value1, Date value2) { - addCriterion("updated_at not between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNull() { - addCriterion("updated_by is null"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNotNull() { - addCriterion("updated_by is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedByEqualTo(String value) { - addCriterion("updated_by =", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotEqualTo(String value) { - addCriterion("updated_by <>", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThan(String value) { - addCriterion("updated_by >", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThanOrEqualTo(String value) { - addCriterion("updated_by >=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThan(String value) { - addCriterion("updated_by <", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThanOrEqualTo(String value) { - addCriterion("updated_by <=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLike(String value) { - addCriterion("updated_by like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotLike(String value) { - addCriterion("updated_by not like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByIn(List values) { - addCriterion("updated_by in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotIn(List values) { - addCriterion("updated_by not in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByBetween(String value1, String value2) { - addCriterion("updated_by between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotBetween(String value1, String value2) { - addCriterion("updated_by not between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andSemanticTypeIsNull() { - addCriterion("semantic_type is null"); - return (Criteria) this; - } - - public Criteria andSemanticTypeIsNotNull() { - addCriterion("semantic_type is not null"); - return (Criteria) this; - } - - public Criteria andSemanticTypeEqualTo(String value) { - addCriterion("semantic_type =", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeNotEqualTo(String value) { - addCriterion("semantic_type <>", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeGreaterThan(String value) { - addCriterion("semantic_type >", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeGreaterThanOrEqualTo(String value) { - addCriterion("semantic_type >=", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeLessThan(String value) { - addCriterion("semantic_type <", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeLessThanOrEqualTo(String value) { - addCriterion("semantic_type <=", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeLike(String value) { - addCriterion("semantic_type like", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeNotLike(String value) { - addCriterion("semantic_type not like", value, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeIn(List values) { - addCriterion("semantic_type in", values, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeNotIn(List values) { - addCriterion("semantic_type not in", values, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeBetween(String value1, String value2) { - addCriterion("semantic_type between", value1, value2, "semanticType"); - return (Criteria) this; - } - - public Criteria andSemanticTypeNotBetween(String value1, String value2) { - addCriterion("semantic_type not between", value1, value2, "semanticType"); - return (Criteria) this; - } - - public Criteria andAliasIsNull() { - addCriterion("alias is null"); - return (Criteria) this; - } - - public Criteria andAliasIsNotNull() { - addCriterion("alias is not null"); - return (Criteria) this; - } - - public Criteria andAliasEqualTo(String value) { - addCriterion("alias =", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotEqualTo(String value) { - addCriterion("alias <>", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThan(String value) { - addCriterion("alias >", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThanOrEqualTo(String value) { - addCriterion("alias >=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThan(String value) { - addCriterion("alias <", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThanOrEqualTo(String value) { - addCriterion("alias <=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLike(String value) { - addCriterion("alias like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotLike(String value) { - addCriterion("alias not like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasIn(List values) { - addCriterion("alias in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotIn(List values) { - addCriterion("alias not in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasBetween(String value1, String value2) { - addCriterion("alias between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotBetween(String value1, String value2) { - addCriterion("alias not between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andDefaultValuesIsNull() { - addCriterion("default_values is null"); - return (Criteria) this; - } - - public Criteria andDefaultValuesIsNotNull() { - addCriterion("default_values is not null"); - return (Criteria) this; - } - - public Criteria andDefaultValuesEqualTo(String value) { - addCriterion("default_values =", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesNotEqualTo(String value) { - addCriterion("default_values <>", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesGreaterThan(String value) { - addCriterion("default_values >", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesGreaterThanOrEqualTo(String value) { - addCriterion("default_values >=", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesLessThan(String value) { - addCriterion("default_values <", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesLessThanOrEqualTo(String value) { - addCriterion("default_values <=", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesLike(String value) { - addCriterion("default_values like", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesNotLike(String value) { - addCriterion("default_values not like", value, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesIn(List values) { - addCriterion("default_values in", values, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesNotIn(List values) { - addCriterion("default_values not in", values, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesBetween(String value1, String value2) { - addCriterion("default_values between", value1, value2, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDefaultValuesNotBetween(String value1, String value2) { - addCriterion("default_values not between", value1, value2, "defaultValues"); - return (Criteria) this; - } - - public Criteria andDimValueMapsIsNull() { - addCriterion("dim_value_maps is null"); - return (Criteria) this; - } - - public Criteria andDimValueMapsIsNotNull() { - addCriterion("dim_value_maps is not null"); - return (Criteria) this; - } - - public Criteria andDimValueMapsEqualTo(String value) { - addCriterion("dim_value_maps =", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsNotEqualTo(String value) { - addCriterion("dim_value_maps <>", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsGreaterThan(String value) { - addCriterion("dim_value_maps >", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsGreaterThanOrEqualTo(String value) { - addCriterion("dim_value_maps >=", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsLessThan(String value) { - addCriterion("dim_value_maps <", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsLessThanOrEqualTo(String value) { - addCriterion("dim_value_maps <=", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsLike(String value) { - addCriterion("dim_value_maps like", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsNotLike(String value) { - addCriterion("dim_value_maps not like", value, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsIn(List values) { - addCriterion("dim_value_maps in", values, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsNotIn(List values) { - addCriterion("dim_value_maps not in", values, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsBetween(String value1, String value2) { - addCriterion("dim_value_maps between", value1, value2, "dimValueMaps"); - return (Criteria) this; - } - - public Criteria andDimValueMapsNotBetween(String value1, String value2) { - addCriterion("dim_value_maps not between", value1, value2, "dimValueMaps"); - return (Criteria) this; - } - } - - /** - * s2_dimension - */ - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - /** - * s2_dimension null - */ - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - } -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DomainDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DomainDO.java index 79e93ce94..f45aa7305 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DomainDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DomainDO.java @@ -1,11 +1,19 @@ package com.tencent.supersonic.semantic.model.domain.dataobject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + import java.util.Date; +@Data +@TableName("s2_domain") public class DomainDO { /** * 自增ID */ + @TableId(type = IdType.AUTO) private Long id; /** @@ -73,227 +81,4 @@ public class DomainDO { */ private String viewOrg; - /** - * 自增ID - * @return id 自增ID - */ - public Long getId() { - return id; - } - - /** - * 自增ID - * @param id 自增ID - */ - public void setId(Long id) { - this.id = id; - } - - /** - * 主题域名称 - * @return name 主题域名称 - */ - public String getName() { - return name; - } - - /** - * 主题域名称 - * @param name 主题域名称 - */ - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - /** - * 内部名称 - * @return biz_name 内部名称 - */ - public String getBizName() { - return bizName; - } - - /** - * 内部名称 - * @param bizName 内部名称 - */ - public void setBizName(String bizName) { - this.bizName = bizName == null ? null : bizName.trim(); - } - - /** - * 父主题域ID - * @return parent_id 父主题域ID - */ - public Long getParentId() { - return parentId; - } - - /** - * 父主题域ID - * @param parentId 父主题域ID - */ - public void setParentId(Long parentId) { - this.parentId = parentId; - } - - /** - * 主题域状态 - * @return status 主题域状态 - */ - public Integer getStatus() { - return status; - } - - /** - * 主题域状态 - * @param status 主题域状态 - */ - public void setStatus(Integer status) { - this.status = status; - } - - /** - * 创建时间 - * @return created_at 创建时间 - */ - public Date getCreatedAt() { - return createdAt; - } - - /** - * 创建时间 - * @param createdAt 创建时间 - */ - public void setCreatedAt(Date createdAt) { - this.createdAt = createdAt; - } - - /** - * 创建人 - * @return created_by 创建人 - */ - public String getCreatedBy() { - return createdBy; - } - - /** - * 创建人 - * @param createdBy 创建人 - */ - public void setCreatedBy(String createdBy) { - this.createdBy = createdBy == null ? null : createdBy.trim(); - } - - /** - * 更新时间 - * @return updated_at 更新时间 - */ - public Date getUpdatedAt() { - return updatedAt; - } - - /** - * 更新时间 - * @param updatedAt 更新时间 - */ - public void setUpdatedAt(Date updatedAt) { - this.updatedAt = updatedAt; - } - - /** - * 更新人 - * @return updated_by 更新人 - */ - public String getUpdatedBy() { - return updatedBy; - } - - /** - * 更新人 - * @param updatedBy 更新人 - */ - public void setUpdatedBy(String updatedBy) { - this.updatedBy = updatedBy == null ? null : updatedBy.trim(); - } - - /** - * 主题域管理员 - * @return admin 主题域管理员 - */ - public String getAdmin() { - return admin; - } - - /** - * 主题域管理员 - * @param admin 主题域管理员 - */ - public void setAdmin(String admin) { - this.admin = admin == null ? null : admin.trim(); - } - - /** - * 主题域管理员组织 - * @return admin_org 主题域管理员组织 - */ - public String getAdminOrg() { - return adminOrg; - } - - /** - * 主题域管理员组织 - * @param adminOrg 主题域管理员组织 - */ - public void setAdminOrg(String adminOrg) { - this.adminOrg = adminOrg == null ? null : adminOrg.trim(); - } - - /** - * 主题域是否公开 - * @return is_open 主题域是否公开 - */ - public Integer getIsOpen() { - return isOpen; - } - - /** - * 主题域是否公开 - * @param isOpen 主题域是否公开 - */ - public void setIsOpen(Integer isOpen) { - this.isOpen = isOpen; - } - - /** - * 主题域可用用户 - * @return viewer 主题域可用用户 - */ - public String getViewer() { - return viewer; - } - - /** - * 主题域可用用户 - * @param viewer 主题域可用用户 - */ - public void setViewer(String viewer) { - this.viewer = viewer == null ? null : viewer.trim(); - } - - /** - * 主题域可用组织 - * @return view_org 主题域可用组织 - */ - public String getViewOrg() { - return viewOrg; - } - - /** - * 主题域可用组织 - * @param viewOrg 主题域可用组织 - */ - public void setViewOrg(String viewOrg) { - this.viewOrg = viewOrg == null ? null : viewOrg.trim(); - } } \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/MetricDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/MetricDOExample.java deleted file mode 100644 index e49e7c06e..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/MetricDOExample.java +++ /dev/null @@ -1,1435 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class MetricDOExample { - /** - * s2_metric - */ - protected String orderByClause; - - /** - * s2_metric - */ - protected boolean distinct; - - /** - * s2_metric - */ - protected List oredCriteria; - - /** - * s2_metric - */ - protected Integer limitStart; - - /** - * s2_metric - */ - protected Integer limitEnd; - - /** - * - * @mbg.generated - */ - public MetricDOExample() { - oredCriteria = new ArrayList(); - } - - /** - * - * @mbg.generated - */ - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - /** - * - * @mbg.generated - */ - public String getOrderByClause() { - return orderByClause; - } - - /** - * - * @mbg.generated - */ - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - /** - * - * @mbg.generated - */ - public boolean isDistinct() { - return distinct; - } - - /** - * - * @mbg.generated - */ - public List getOredCriteria() { - return oredCriteria; - } - - /** - * - * @mbg.generated - */ - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - /** - * - * @mbg.generated - */ - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - /** - * - * @mbg.generated - */ - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - /** - * - * @mbg.generated - */ - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - /** - * - * @mbg.generated - */ - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - /** - * - * @mbg.generated - */ - public void setLimitStart(Integer limitStart) { - this.limitStart=limitStart; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitStart() { - return limitStart; - } - - /** - * - * @mbg.generated - */ - public void setLimitEnd(Integer limitEnd) { - this.limitEnd=limitEnd; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitEnd() { - return limitEnd; - } - - /** - * s2_metric null - */ - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andModelIdIsNull() { - addCriterion("model_id is null"); - return (Criteria) this; - } - - public Criteria andModelIdIsNotNull() { - addCriterion("model_id is not null"); - return (Criteria) this; - } - - public Criteria andModelIdEqualTo(Long value) { - addCriterion("model_id =", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotEqualTo(Long value) { - addCriterion("model_id <>", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThan(Long value) { - addCriterion("model_id >", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdGreaterThanOrEqualTo(Long value) { - addCriterion("model_id >=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThan(Long value) { - addCriterion("model_id <", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdLessThanOrEqualTo(Long value) { - addCriterion("model_id <=", value, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdIn(List values) { - addCriterion("model_id in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotIn(List values) { - addCriterion("model_id not in", values, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdBetween(Long value1, Long value2) { - addCriterion("model_id between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andModelIdNotBetween(Long value1, Long value2) { - addCriterion("model_id not between", value1, value2, "modelId"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andBizNameIsNull() { - addCriterion("biz_name is null"); - return (Criteria) this; - } - - public Criteria andBizNameIsNotNull() { - addCriterion("biz_name is not null"); - return (Criteria) this; - } - - public Criteria andBizNameEqualTo(String value) { - addCriterion("biz_name =", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotEqualTo(String value) { - addCriterion("biz_name <>", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThan(String value) { - addCriterion("biz_name >", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThanOrEqualTo(String value) { - addCriterion("biz_name >=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThan(String value) { - addCriterion("biz_name <", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThanOrEqualTo(String value) { - addCriterion("biz_name <=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLike(String value) { - addCriterion("biz_name like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotLike(String value) { - addCriterion("biz_name not like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameIn(List values) { - addCriterion("biz_name in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotIn(List values) { - addCriterion("biz_name not in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameBetween(String value1, String value2) { - addCriterion("biz_name between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotBetween(String value1, String value2) { - addCriterion("biz_name not between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNull() { - addCriterion("description is null"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNotNull() { - addCriterion("description is not null"); - return (Criteria) this; - } - - public Criteria andDescriptionEqualTo(String value) { - addCriterion("description =", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotEqualTo(String value) { - addCriterion("description <>", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThan(String value) { - addCriterion("description >", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThanOrEqualTo(String value) { - addCriterion("description >=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThan(String value) { - addCriterion("description <", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThanOrEqualTo(String value) { - addCriterion("description <=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLike(String value) { - addCriterion("description like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotLike(String value) { - addCriterion("description not like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionIn(List values) { - addCriterion("description in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotIn(List values) { - addCriterion("description not in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionBetween(String value1, String value2) { - addCriterion("description between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotBetween(String value1, String value2) { - addCriterion("description not between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(Integer value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(Integer value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(Integer value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(Integer value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(Integer value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(Integer value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(Integer value1, Integer value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(Integer value1, Integer value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIsNull() { - addCriterion("sensitive_level is null"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIsNotNull() { - addCriterion("sensitive_level is not null"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelEqualTo(Integer value) { - addCriterion("sensitive_level =", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotEqualTo(Integer value) { - addCriterion("sensitive_level <>", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelGreaterThan(Integer value) { - addCriterion("sensitive_level >", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelGreaterThanOrEqualTo(Integer value) { - addCriterion("sensitive_level >=", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelLessThan(Integer value) { - addCriterion("sensitive_level <", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelLessThanOrEqualTo(Integer value) { - addCriterion("sensitive_level <=", value, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelIn(List values) { - addCriterion("sensitive_level in", values, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotIn(List values) { - addCriterion("sensitive_level not in", values, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelBetween(Integer value1, Integer value2) { - addCriterion("sensitive_level between", value1, value2, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andSensitiveLevelNotBetween(Integer value1, Integer value2) { - addCriterion("sensitive_level not between", value1, value2, "sensitiveLevel"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNull() { - addCriterion("created_at is null"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNotNull() { - addCriterion("created_at is not null"); - return (Criteria) this; - } - - public Criteria andCreatedAtEqualTo(Date value) { - addCriterion("created_at =", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotEqualTo(Date value) { - addCriterion("created_at <>", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThan(Date value) { - addCriterion("created_at >", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("created_at >=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThan(Date value) { - addCriterion("created_at <", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThanOrEqualTo(Date value) { - addCriterion("created_at <=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtIn(List values) { - addCriterion("created_at in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotIn(List values) { - addCriterion("created_at not in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtBetween(Date value1, Date value2) { - addCriterion("created_at between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotBetween(Date value1, Date value2) { - addCriterion("created_at not between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNull() { - addCriterion("created_by is null"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNotNull() { - addCriterion("created_by is not null"); - return (Criteria) this; - } - - public Criteria andCreatedByEqualTo(String value) { - addCriterion("created_by =", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotEqualTo(String value) { - addCriterion("created_by <>", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThan(String value) { - addCriterion("created_by >", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThanOrEqualTo(String value) { - addCriterion("created_by >=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThan(String value) { - addCriterion("created_by <", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThanOrEqualTo(String value) { - addCriterion("created_by <=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLike(String value) { - addCriterion("created_by like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotLike(String value) { - addCriterion("created_by not like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByIn(List values) { - addCriterion("created_by in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotIn(List values) { - addCriterion("created_by not in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByBetween(String value1, String value2) { - addCriterion("created_by between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotBetween(String value1, String value2) { - addCriterion("created_by not between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNull() { - addCriterion("updated_at is null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNotNull() { - addCriterion("updated_at is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtEqualTo(Date value) { - addCriterion("updated_at =", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotEqualTo(Date value) { - addCriterion("updated_at <>", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThan(Date value) { - addCriterion("updated_at >", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("updated_at >=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThan(Date value) { - addCriterion("updated_at <", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThanOrEqualTo(Date value) { - addCriterion("updated_at <=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIn(List values) { - addCriterion("updated_at in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotIn(List values) { - addCriterion("updated_at not in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtBetween(Date value1, Date value2) { - addCriterion("updated_at between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotBetween(Date value1, Date value2) { - addCriterion("updated_at not between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNull() { - addCriterion("updated_by is null"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNotNull() { - addCriterion("updated_by is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedByEqualTo(String value) { - addCriterion("updated_by =", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotEqualTo(String value) { - addCriterion("updated_by <>", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThan(String value) { - addCriterion("updated_by >", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThanOrEqualTo(String value) { - addCriterion("updated_by >=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThan(String value) { - addCriterion("updated_by <", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThanOrEqualTo(String value) { - addCriterion("updated_by <=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLike(String value) { - addCriterion("updated_by like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotLike(String value) { - addCriterion("updated_by not like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByIn(List values) { - addCriterion("updated_by in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotIn(List values) { - addCriterion("updated_by not in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByBetween(String value1, String value2) { - addCriterion("updated_by between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotBetween(String value1, String value2) { - addCriterion("updated_by not between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeIsNull() { - addCriterion("data_format_type is null"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeIsNotNull() { - addCriterion("data_format_type is not null"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeEqualTo(String value) { - addCriterion("data_format_type =", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeNotEqualTo(String value) { - addCriterion("data_format_type <>", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeGreaterThan(String value) { - addCriterion("data_format_type >", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeGreaterThanOrEqualTo(String value) { - addCriterion("data_format_type >=", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeLessThan(String value) { - addCriterion("data_format_type <", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeLessThanOrEqualTo(String value) { - addCriterion("data_format_type <=", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeLike(String value) { - addCriterion("data_format_type like", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeNotLike(String value) { - addCriterion("data_format_type not like", value, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeIn(List values) { - addCriterion("data_format_type in", values, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeNotIn(List values) { - addCriterion("data_format_type not in", values, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeBetween(String value1, String value2) { - addCriterion("data_format_type between", value1, value2, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatTypeNotBetween(String value1, String value2) { - addCriterion("data_format_type not between", value1, value2, "dataFormatType"); - return (Criteria) this; - } - - public Criteria andDataFormatIsNull() { - addCriterion("data_format is null"); - return (Criteria) this; - } - - public Criteria andDataFormatIsNotNull() { - addCriterion("data_format is not null"); - return (Criteria) this; - } - - public Criteria andDataFormatEqualTo(String value) { - addCriterion("data_format =", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatNotEqualTo(String value) { - addCriterion("data_format <>", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatGreaterThan(String value) { - addCriterion("data_format >", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatGreaterThanOrEqualTo(String value) { - addCriterion("data_format >=", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatLessThan(String value) { - addCriterion("data_format <", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatLessThanOrEqualTo(String value) { - addCriterion("data_format <=", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatLike(String value) { - addCriterion("data_format like", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatNotLike(String value) { - addCriterion("data_format not like", value, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatIn(List values) { - addCriterion("data_format in", values, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatNotIn(List values) { - addCriterion("data_format not in", values, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatBetween(String value1, String value2) { - addCriterion("data_format between", value1, value2, "dataFormat"); - return (Criteria) this; - } - - public Criteria andDataFormatNotBetween(String value1, String value2) { - addCriterion("data_format not between", value1, value2, "dataFormat"); - return (Criteria) this; - } - - public Criteria andAliasIsNull() { - addCriterion("alias is null"); - return (Criteria) this; - } - - public Criteria andAliasIsNotNull() { - addCriterion("alias is not null"); - return (Criteria) this; - } - - public Criteria andAliasEqualTo(String value) { - addCriterion("alias =", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotEqualTo(String value) { - addCriterion("alias <>", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThan(String value) { - addCriterion("alias >", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThanOrEqualTo(String value) { - addCriterion("alias >=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThan(String value) { - addCriterion("alias <", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThanOrEqualTo(String value) { - addCriterion("alias <=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLike(String value) { - addCriterion("alias like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotLike(String value) { - addCriterion("alias not like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasIn(List values) { - addCriterion("alias in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotIn(List values) { - addCriterion("alias not in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasBetween(String value1, String value2) { - addCriterion("alias between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotBetween(String value1, String value2) { - addCriterion("alias not between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andTagsIsNull() { - addCriterion("tags is null"); - return (Criteria) this; - } - - public Criteria andTagsIsNotNull() { - addCriterion("tags is not null"); - return (Criteria) this; - } - - public Criteria andTagsEqualTo(String value) { - addCriterion("tags =", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsNotEqualTo(String value) { - addCriterion("tags <>", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsGreaterThan(String value) { - addCriterion("tags >", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsGreaterThanOrEqualTo(String value) { - addCriterion("tags >=", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsLessThan(String value) { - addCriterion("tags <", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsLessThanOrEqualTo(String value) { - addCriterion("tags <=", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsLike(String value) { - addCriterion("tags like", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsNotLike(String value) { - addCriterion("tags not like", value, "tags"); - return (Criteria) this; - } - - public Criteria andTagsIn(List values) { - addCriterion("tags in", values, "tags"); - return (Criteria) this; - } - - public Criteria andTagsNotIn(List values) { - addCriterion("tags not in", values, "tags"); - return (Criteria) this; - } - - public Criteria andTagsBetween(String value1, String value2) { - addCriterion("tags between", value1, value2, "tags"); - return (Criteria) this; - } - - public Criteria andTagsNotBetween(String value1, String value2) { - addCriterion("tags not between", value1, value2, "tags"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsIsNull() { - addCriterion("relate_dimensions is null"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsIsNotNull() { - addCriterion("relate_dimensions is not null"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsEqualTo(String value) { - addCriterion("relate_dimensions =", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsNotEqualTo(String value) { - addCriterion("relate_dimensions <>", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsGreaterThan(String value) { - addCriterion("relate_dimensions >", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsGreaterThanOrEqualTo(String value) { - addCriterion("relate_dimensions >=", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsLessThan(String value) { - addCriterion("relate_dimensions <", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsLessThanOrEqualTo(String value) { - addCriterion("relate_dimensions <=", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsLike(String value) { - addCriterion("relate_dimensions like", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsNotLike(String value) { - addCriterion("relate_dimensions not like", value, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsIn(List values) { - addCriterion("relate_dimensions in", values, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsNotIn(List values) { - addCriterion("relate_dimensions not in", values, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsBetween(String value1, String value2) { - addCriterion("relate_dimensions between", value1, value2, "relateDimensions"); - return (Criteria) this; - } - - public Criteria andRelateDimensionsNotBetween(String value1, String value2) { - addCriterion("relate_dimensions not between", value1, value2, "relateDimensions"); - return (Criteria) this; - } - } - - /** - * s2_metric - */ - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - /** - * s2_metric null - */ - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java index e1787906e..8f6643289 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java @@ -1,383 +1,56 @@ package com.tencent.supersonic.semantic.model.domain.dataobject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; import java.util.Date; +@Data +@TableName("s2_model") public class ModelDO { - /** - * - */ + + @TableId(value = "id", type = IdType.AUTO) private Long id; - /** - * - */ - private String name; - - /** - * - */ - private String bizName; - - /** - * - */ private Long domainId; - /** - * - */ - private String alias; + private String name; + + private String bizName; - /** - * - */ private String description; - /** - * - */ - private String viewer; + private Long databaseId; - /** - * - */ - private String viewOrg; - - /** - * - */ - private String admin; - - /** - * - */ - private String adminOrg; - - /** - * - */ - private Integer isOpen; - - /** - * - */ - private String createdBy; - - /** - * - */ - private Date createdAt; - - /** - * - */ - private String updatedBy; - - /** - * - */ - private Date updatedAt; - - /** - * - */ - private String drillDownDimensions; - - /** - * - */ private Integer status; - /** - * - */ - private String entity; + private Date createdAt; - /** - * - * @return id - */ - public Long getId() { - return id; - } + private String createdBy; - /** - * - * @param id - */ - public void setId(Long id) { - this.id = id; - } + private Date updatedAt; - /** - * - * @return name - */ - public String getName() { - return name; - } + private String updatedBy; - /** - * - * @param name - */ - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } + private String modelDetail; - /** - * - * @return biz_name - */ - public String getBizName() { - return bizName; - } + private String depends; - /** - * - * @param bizName - */ - public void setBizName(String bizName) { - this.bizName = bizName == null ? null : bizName.trim(); - } + private String filterSql; - /** - * - * @return domain_id - */ - public Long getDomainId() { - return domainId; - } + private String viewer; - /** - * - * @param domainId - */ - public void setDomainId(Long domainId) { - this.domainId = domainId; - } + private String viewOrg; - /** - * - * @return alias - */ - public String getAlias() { - return alias; - } + private String admin; - /** - * - * @param alias - */ - public void setAlias(String alias) { - this.alias = alias == null ? null : alias.trim(); - } + private String adminOrg; - /** - * - * @return description - */ - public String getDescription() { - return description; - } + private Integer isOpen; - /** - * - * @param description - */ - public void setDescription(String description) { - this.description = description == null ? null : description.trim(); - } + private String drillDownDimensions; - /** - * - * @return viewer - */ - public String getViewer() { - return viewer; - } + private String alias; - /** - * - * @param viewer - */ - public void setViewer(String viewer) { - this.viewer = viewer == null ? null : viewer.trim(); - } - - /** - * - * @return view_org - */ - public String getViewOrg() { - return viewOrg; - } - - /** - * - * @param viewOrg - */ - public void setViewOrg(String viewOrg) { - this.viewOrg = viewOrg == null ? null : viewOrg.trim(); - } - - /** - * - * @return admin - */ - public String getAdmin() { - return admin; - } - - /** - * - * @param admin - */ - public void setAdmin(String admin) { - this.admin = admin == null ? null : admin.trim(); - } - - /** - * - * @return admin_org - */ - public String getAdminOrg() { - return adminOrg; - } - - /** - * - * @param adminOrg - */ - public void setAdminOrg(String adminOrg) { - this.adminOrg = adminOrg == null ? null : adminOrg.trim(); - } - - /** - * - * @return is_open - */ - public Integer getIsOpen() { - return isOpen; - } - - /** - * - * @param isOpen - */ - public void setIsOpen(Integer isOpen) { - this.isOpen = isOpen; - } - - /** - * - * @return created_by - */ - public String getCreatedBy() { - return createdBy; - } - - /** - * - * @param createdBy - */ - public void setCreatedBy(String createdBy) { - this.createdBy = createdBy == null ? null : createdBy.trim(); - } - - /** - * - * @return created_at - */ - public Date getCreatedAt() { - return createdAt; - } - - /** - * - * @param createdAt - */ - public void setCreatedAt(Date createdAt) { - this.createdAt = createdAt; - } - - /** - * - * @return updated_by - */ - public String getUpdatedBy() { - return updatedBy; - } - - /** - * - * @param updatedBy - */ - public void setUpdatedBy(String updatedBy) { - this.updatedBy = updatedBy == null ? null : updatedBy.trim(); - } - - /** - * - * @return updated_at - */ - public Date getUpdatedAt() { - return updatedAt; - } - - /** - * - * @param updatedAt - */ - public void setUpdatedAt(Date updatedAt) { - this.updatedAt = updatedAt; - } - - /** - * - * @return drill_down_dimensions - */ - public String getDrillDownDimensions() { - return drillDownDimensions; - } - - /** - * - * @param drillDownDimensions - */ - public void setDrillDownDimensions(String drillDownDimensions) { - this.drillDownDimensions = drillDownDimensions == null ? null : drillDownDimensions.trim(); - } - - /** - * - * @return status - */ - public Integer getStatus() { - return status; - } - - /** - * - * @param status - */ - public void setStatus(Integer status) { - this.status = status; - } - - /** - * - * @return entity - */ - public String getEntity() { - return entity; - } - - /** - * - * @param entity - */ - public void setEntity(String entity) { - this.entity = entity == null ? null : entity.trim(); - } } \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java deleted file mode 100644 index 8204278fa..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java +++ /dev/null @@ -1,1435 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class ModelDOExample { - /** - * s2_model - */ - protected String orderByClause; - - /** - * s2_model - */ - protected boolean distinct; - - /** - * s2_model - */ - protected List oredCriteria; - - /** - * s2_model - */ - protected Integer limitStart; - - /** - * s2_model - */ - protected Integer limitEnd; - - /** - * - * @mbg.generated - */ - public ModelDOExample() { - oredCriteria = new ArrayList(); - } - - /** - * - * @mbg.generated - */ - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - /** - * - * @mbg.generated - */ - public String getOrderByClause() { - return orderByClause; - } - - /** - * - * @mbg.generated - */ - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - /** - * - * @mbg.generated - */ - public boolean isDistinct() { - return distinct; - } - - /** - * - * @mbg.generated - */ - public List getOredCriteria() { - return oredCriteria; - } - - /** - * - * @mbg.generated - */ - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - /** - * - * @mbg.generated - */ - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - /** - * - * @mbg.generated - */ - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - /** - * - * @mbg.generated - */ - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - /** - * - * @mbg.generated - */ - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - /** - * - * @mbg.generated - */ - public void setLimitStart(Integer limitStart) { - this.limitStart=limitStart; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitStart() { - return limitStart; - } - - /** - * - * @mbg.generated - */ - public void setLimitEnd(Integer limitEnd) { - this.limitEnd=limitEnd; - } - - /** - * - * @mbg.generated - */ - public Integer getLimitEnd() { - return limitEnd; - } - - /** - * s2_model null - */ - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andBizNameIsNull() { - addCriterion("biz_name is null"); - return (Criteria) this; - } - - public Criteria andBizNameIsNotNull() { - addCriterion("biz_name is not null"); - return (Criteria) this; - } - - public Criteria andBizNameEqualTo(String value) { - addCriterion("biz_name =", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotEqualTo(String value) { - addCriterion("biz_name <>", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThan(String value) { - addCriterion("biz_name >", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameGreaterThanOrEqualTo(String value) { - addCriterion("biz_name >=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThan(String value) { - addCriterion("biz_name <", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLessThanOrEqualTo(String value) { - addCriterion("biz_name <=", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameLike(String value) { - addCriterion("biz_name like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotLike(String value) { - addCriterion("biz_name not like", value, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameIn(List values) { - addCriterion("biz_name in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotIn(List values) { - addCriterion("biz_name not in", values, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameBetween(String value1, String value2) { - addCriterion("biz_name between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andBizNameNotBetween(String value1, String value2) { - addCriterion("biz_name not between", value1, value2, "bizName"); - return (Criteria) this; - } - - public Criteria andDomainIdIsNull() { - addCriterion("domain_id is null"); - return (Criteria) this; - } - - public Criteria andDomainIdIsNotNull() { - addCriterion("domain_id is not null"); - return (Criteria) this; - } - - public Criteria andDomainIdEqualTo(Long value) { - addCriterion("domain_id =", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdNotEqualTo(Long value) { - addCriterion("domain_id <>", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdGreaterThan(Long value) { - addCriterion("domain_id >", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdGreaterThanOrEqualTo(Long value) { - addCriterion("domain_id >=", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdLessThan(Long value) { - addCriterion("domain_id <", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdLessThanOrEqualTo(Long value) { - addCriterion("domain_id <=", value, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdIn(List values) { - addCriterion("domain_id in", values, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdNotIn(List values) { - addCriterion("domain_id not in", values, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdBetween(Long value1, Long value2) { - addCriterion("domain_id between", value1, value2, "domainId"); - return (Criteria) this; - } - - public Criteria andDomainIdNotBetween(Long value1, Long value2) { - addCriterion("domain_id not between", value1, value2, "domainId"); - return (Criteria) this; - } - - public Criteria andAliasIsNull() { - addCriterion("alias is null"); - return (Criteria) this; - } - - public Criteria andAliasIsNotNull() { - addCriterion("alias is not null"); - return (Criteria) this; - } - - public Criteria andAliasEqualTo(String value) { - addCriterion("alias =", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotEqualTo(String value) { - addCriterion("alias <>", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThan(String value) { - addCriterion("alias >", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasGreaterThanOrEqualTo(String value) { - addCriterion("alias >=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThan(String value) { - addCriterion("alias <", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLessThanOrEqualTo(String value) { - addCriterion("alias <=", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasLike(String value) { - addCriterion("alias like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotLike(String value) { - addCriterion("alias not like", value, "alias"); - return (Criteria) this; - } - - public Criteria andAliasIn(List values) { - addCriterion("alias in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotIn(List values) { - addCriterion("alias not in", values, "alias"); - return (Criteria) this; - } - - public Criteria andAliasBetween(String value1, String value2) { - addCriterion("alias between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andAliasNotBetween(String value1, String value2) { - addCriterion("alias not between", value1, value2, "alias"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNull() { - addCriterion("description is null"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNotNull() { - addCriterion("description is not null"); - return (Criteria) this; - } - - public Criteria andDescriptionEqualTo(String value) { - addCriterion("description =", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotEqualTo(String value) { - addCriterion("description <>", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThan(String value) { - addCriterion("description >", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThanOrEqualTo(String value) { - addCriterion("description >=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThan(String value) { - addCriterion("description <", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThanOrEqualTo(String value) { - addCriterion("description <=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLike(String value) { - addCriterion("description like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotLike(String value) { - addCriterion("description not like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionIn(List values) { - addCriterion("description in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotIn(List values) { - addCriterion("description not in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionBetween(String value1, String value2) { - addCriterion("description between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotBetween(String value1, String value2) { - addCriterion("description not between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andViewerIsNull() { - addCriterion("viewer is null"); - return (Criteria) this; - } - - public Criteria andViewerIsNotNull() { - addCriterion("viewer is not null"); - return (Criteria) this; - } - - public Criteria andViewerEqualTo(String value) { - addCriterion("viewer =", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerNotEqualTo(String value) { - addCriterion("viewer <>", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerGreaterThan(String value) { - addCriterion("viewer >", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerGreaterThanOrEqualTo(String value) { - addCriterion("viewer >=", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerLessThan(String value) { - addCriterion("viewer <", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerLessThanOrEqualTo(String value) { - addCriterion("viewer <=", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerLike(String value) { - addCriterion("viewer like", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerNotLike(String value) { - addCriterion("viewer not like", value, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerIn(List values) { - addCriterion("viewer in", values, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerNotIn(List values) { - addCriterion("viewer not in", values, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerBetween(String value1, String value2) { - addCriterion("viewer between", value1, value2, "viewer"); - return (Criteria) this; - } - - public Criteria andViewerNotBetween(String value1, String value2) { - addCriterion("viewer not between", value1, value2, "viewer"); - return (Criteria) this; - } - - public Criteria andViewOrgIsNull() { - addCriterion("view_org is null"); - return (Criteria) this; - } - - public Criteria andViewOrgIsNotNull() { - addCriterion("view_org is not null"); - return (Criteria) this; - } - - public Criteria andViewOrgEqualTo(String value) { - addCriterion("view_org =", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgNotEqualTo(String value) { - addCriterion("view_org <>", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgGreaterThan(String value) { - addCriterion("view_org >", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgGreaterThanOrEqualTo(String value) { - addCriterion("view_org >=", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgLessThan(String value) { - addCriterion("view_org <", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgLessThanOrEqualTo(String value) { - addCriterion("view_org <=", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgLike(String value) { - addCriterion("view_org like", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgNotLike(String value) { - addCriterion("view_org not like", value, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgIn(List values) { - addCriterion("view_org in", values, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgNotIn(List values) { - addCriterion("view_org not in", values, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgBetween(String value1, String value2) { - addCriterion("view_org between", value1, value2, "viewOrg"); - return (Criteria) this; - } - - public Criteria andViewOrgNotBetween(String value1, String value2) { - addCriterion("view_org not between", value1, value2, "viewOrg"); - return (Criteria) this; - } - - public Criteria andAdminIsNull() { - addCriterion("admin is null"); - return (Criteria) this; - } - - public Criteria andAdminIsNotNull() { - addCriterion("admin is not null"); - return (Criteria) this; - } - - public Criteria andAdminEqualTo(String value) { - addCriterion("admin =", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminNotEqualTo(String value) { - addCriterion("admin <>", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminGreaterThan(String value) { - addCriterion("admin >", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminGreaterThanOrEqualTo(String value) { - addCriterion("admin >=", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminLessThan(String value) { - addCriterion("admin <", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminLessThanOrEqualTo(String value) { - addCriterion("admin <=", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminLike(String value) { - addCriterion("admin like", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminNotLike(String value) { - addCriterion("admin not like", value, "admin"); - return (Criteria) this; - } - - public Criteria andAdminIn(List values) { - addCriterion("admin in", values, "admin"); - return (Criteria) this; - } - - public Criteria andAdminNotIn(List values) { - addCriterion("admin not in", values, "admin"); - return (Criteria) this; - } - - public Criteria andAdminBetween(String value1, String value2) { - addCriterion("admin between", value1, value2, "admin"); - return (Criteria) this; - } - - public Criteria andAdminNotBetween(String value1, String value2) { - addCriterion("admin not between", value1, value2, "admin"); - return (Criteria) this; - } - - public Criteria andAdminOrgIsNull() { - addCriterion("admin_org is null"); - return (Criteria) this; - } - - public Criteria andAdminOrgIsNotNull() { - addCriterion("admin_org is not null"); - return (Criteria) this; - } - - public Criteria andAdminOrgEqualTo(String value) { - addCriterion("admin_org =", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgNotEqualTo(String value) { - addCriterion("admin_org <>", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgGreaterThan(String value) { - addCriterion("admin_org >", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgGreaterThanOrEqualTo(String value) { - addCriterion("admin_org >=", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgLessThan(String value) { - addCriterion("admin_org <", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgLessThanOrEqualTo(String value) { - addCriterion("admin_org <=", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgLike(String value) { - addCriterion("admin_org like", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgNotLike(String value) { - addCriterion("admin_org not like", value, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgIn(List values) { - addCriterion("admin_org in", values, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgNotIn(List values) { - addCriterion("admin_org not in", values, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgBetween(String value1, String value2) { - addCriterion("admin_org between", value1, value2, "adminOrg"); - return (Criteria) this; - } - - public Criteria andAdminOrgNotBetween(String value1, String value2) { - addCriterion("admin_org not between", value1, value2, "adminOrg"); - return (Criteria) this; - } - - public Criteria andIsOpenIsNull() { - addCriterion("is_open is null"); - return (Criteria) this; - } - - public Criteria andIsOpenIsNotNull() { - addCriterion("is_open is not null"); - return (Criteria) this; - } - - public Criteria andIsOpenEqualTo(Integer value) { - addCriterion("is_open =", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenNotEqualTo(Integer value) { - addCriterion("is_open <>", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenGreaterThan(Integer value) { - addCriterion("is_open >", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenGreaterThanOrEqualTo(Integer value) { - addCriterion("is_open >=", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenLessThan(Integer value) { - addCriterion("is_open <", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenLessThanOrEqualTo(Integer value) { - addCriterion("is_open <=", value, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenIn(List values) { - addCriterion("is_open in", values, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenNotIn(List values) { - addCriterion("is_open not in", values, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenBetween(Integer value1, Integer value2) { - addCriterion("is_open between", value1, value2, "isOpen"); - return (Criteria) this; - } - - public Criteria andIsOpenNotBetween(Integer value1, Integer value2) { - addCriterion("is_open not between", value1, value2, "isOpen"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNull() { - addCriterion("created_by is null"); - return (Criteria) this; - } - - public Criteria andCreatedByIsNotNull() { - addCriterion("created_by is not null"); - return (Criteria) this; - } - - public Criteria andCreatedByEqualTo(String value) { - addCriterion("created_by =", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotEqualTo(String value) { - addCriterion("created_by <>", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThan(String value) { - addCriterion("created_by >", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByGreaterThanOrEqualTo(String value) { - addCriterion("created_by >=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThan(String value) { - addCriterion("created_by <", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLessThanOrEqualTo(String value) { - addCriterion("created_by <=", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByLike(String value) { - addCriterion("created_by like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotLike(String value) { - addCriterion("created_by not like", value, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByIn(List values) { - addCriterion("created_by in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotIn(List values) { - addCriterion("created_by not in", values, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByBetween(String value1, String value2) { - addCriterion("created_by between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedByNotBetween(String value1, String value2) { - addCriterion("created_by not between", value1, value2, "createdBy"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNull() { - addCriterion("created_at is null"); - return (Criteria) this; - } - - public Criteria andCreatedAtIsNotNull() { - addCriterion("created_at is not null"); - return (Criteria) this; - } - - public Criteria andCreatedAtEqualTo(Date value) { - addCriterion("created_at =", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotEqualTo(Date value) { - addCriterion("created_at <>", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThan(Date value) { - addCriterion("created_at >", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("created_at >=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThan(Date value) { - addCriterion("created_at <", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtLessThanOrEqualTo(Date value) { - addCriterion("created_at <=", value, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtIn(List values) { - addCriterion("created_at in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotIn(List values) { - addCriterion("created_at not in", values, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtBetween(Date value1, Date value2) { - addCriterion("created_at between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andCreatedAtNotBetween(Date value1, Date value2) { - addCriterion("created_at not between", value1, value2, "createdAt"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNull() { - addCriterion("updated_by is null"); - return (Criteria) this; - } - - public Criteria andUpdatedByIsNotNull() { - addCriterion("updated_by is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedByEqualTo(String value) { - addCriterion("updated_by =", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotEqualTo(String value) { - addCriterion("updated_by <>", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThan(String value) { - addCriterion("updated_by >", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByGreaterThanOrEqualTo(String value) { - addCriterion("updated_by >=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThan(String value) { - addCriterion("updated_by <", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLessThanOrEqualTo(String value) { - addCriterion("updated_by <=", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByLike(String value) { - addCriterion("updated_by like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotLike(String value) { - addCriterion("updated_by not like", value, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByIn(List values) { - addCriterion("updated_by in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotIn(List values) { - addCriterion("updated_by not in", values, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByBetween(String value1, String value2) { - addCriterion("updated_by between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedByNotBetween(String value1, String value2) { - addCriterion("updated_by not between", value1, value2, "updatedBy"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNull() { - addCriterion("updated_at is null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIsNotNull() { - addCriterion("updated_at is not null"); - return (Criteria) this; - } - - public Criteria andUpdatedAtEqualTo(Date value) { - addCriterion("updated_at =", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotEqualTo(Date value) { - addCriterion("updated_at <>", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThan(Date value) { - addCriterion("updated_at >", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) { - addCriterion("updated_at >=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThan(Date value) { - addCriterion("updated_at <", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtLessThanOrEqualTo(Date value) { - addCriterion("updated_at <=", value, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtIn(List values) { - addCriterion("updated_at in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotIn(List values) { - addCriterion("updated_at not in", values, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtBetween(Date value1, Date value2) { - addCriterion("updated_at between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andUpdatedAtNotBetween(Date value1, Date value2) { - addCriterion("updated_at not between", value1, value2, "updatedAt"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsIsNull() { - addCriterion("drill_down_dimensions is null"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsIsNotNull() { - addCriterion("drill_down_dimensions is not null"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsEqualTo(String value) { - addCriterion("drill_down_dimensions =", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsNotEqualTo(String value) { - addCriterion("drill_down_dimensions <>", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsGreaterThan(String value) { - addCriterion("drill_down_dimensions >", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsGreaterThanOrEqualTo(String value) { - addCriterion("drill_down_dimensions >=", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsLessThan(String value) { - addCriterion("drill_down_dimensions <", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsLessThanOrEqualTo(String value) { - addCriterion("drill_down_dimensions <=", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsLike(String value) { - addCriterion("drill_down_dimensions like", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsNotLike(String value) { - addCriterion("drill_down_dimensions not like", value, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsIn(List values) { - addCriterion("drill_down_dimensions in", values, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsNotIn(List values) { - addCriterion("drill_down_dimensions not in", values, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsBetween(String value1, String value2) { - addCriterion("drill_down_dimensions between", value1, value2, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andDrillDownDimensionsNotBetween(String value1, String value2) { - addCriterion("drill_down_dimensions not between", value1, value2, "drillDownDimensions"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(Integer value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(Integer value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(Integer value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(Integer value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(Integer value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(Integer value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(Integer value1, Integer value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(Integer value1, Integer value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - } - - /** - * s2_model - */ - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - /** - * s2_model null - */ - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelRelaDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelRelaDO.java new file mode 100644 index 000000000..b9a074274 --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelRelaDO.java @@ -0,0 +1,26 @@ +package com.tencent.supersonic.semantic.model.domain.dataobject; + + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +@Data +@TableName("s2_model_rela") +public class ModelRelaDO { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long domainId; + + private Long fromModelId; + + private Long toModelId; + + private String joinType; + + private String joinCondition; + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java index 6e0b6a775..69b71dafd 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/manager/DatasourceYamlManager.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.semantic.model.domain.manager; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl; @@ -27,27 +27,27 @@ import org.springframework.util.CollectionUtils; @Slf4j public class DatasourceYamlManager { - public static DatasourceYamlTpl convert2YamlObj(Datasource datasource, DatabaseResp databaseResp) { - DatasourceDetail datasourceDetail = datasource.getDatasourceDetail(); + public static DataModelYamlTpl convert2YamlObj(Datasource datasource, DatabaseResp databaseResp) { + ModelDetail datasourceDetail = datasource.getDatasourceDetail(); EngineAdaptor engineAdaptor = EngineAdaptorFactory.getEngineAdaptor(databaseResp.getType()); SysTimeDimensionBuilder.addSysTimeDimension(datasourceDetail.getDimensions(), engineAdaptor); addInterCntMetric(datasource.getBizName(), datasourceDetail); - DatasourceYamlTpl datasourceYamlTpl = new DatasourceYamlTpl(); - BeanUtils.copyProperties(datasourceDetail, datasourceYamlTpl); - datasourceYamlTpl.setIdentifiers(datasourceDetail.getIdentifiers().stream().map(DatasourceYamlManager::convert) + DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl(); + BeanUtils.copyProperties(datasourceDetail, dataModelYamlTpl); + dataModelYamlTpl.setIdentifiers(datasourceDetail.getIdentifiers().stream().map(DatasourceYamlManager::convert) .collect(Collectors.toList())); - datasourceYamlTpl.setDimensions(datasourceDetail.getDimensions().stream().map(DatasourceYamlManager::convert) + dataModelYamlTpl.setDimensions(datasourceDetail.getDimensions().stream().map(DatasourceYamlManager::convert) .collect(Collectors.toList())); - datasourceYamlTpl.setMeasures(datasourceDetail.getMeasures().stream().map(DatasourceYamlManager::convert) + dataModelYamlTpl.setMeasures(datasourceDetail.getMeasures().stream().map(DatasourceYamlManager::convert) .collect(Collectors.toList())); - datasourceYamlTpl.setName(datasource.getBizName()); - datasourceYamlTpl.setSourceId(datasource.getDatabaseId()); + dataModelYamlTpl.setName(datasource.getBizName()); + dataModelYamlTpl.setSourceId(datasource.getDatabaseId()); if (datasourceDetail.getQueryType().equalsIgnoreCase(DatasourceQueryEnum.SQL_QUERY.getName())) { - datasourceYamlTpl.setSqlQuery(datasourceDetail.getSqlQuery()); + dataModelYamlTpl.setSqlQuery(datasourceDetail.getSqlQuery()); } else { - datasourceYamlTpl.setTableQuery(datasourceDetail.getTableQuery()); + dataModelYamlTpl.setTableQuery(datasourceDetail.getTableQuery()); } - return datasourceYamlTpl; + return dataModelYamlTpl; } public static DimensionYamlTpl convert(Dim dim) { @@ -81,7 +81,7 @@ public class DatasourceYamlManager { } - private static void addInterCntMetric(String datasourceEnName, DatasourceDetail datasourceDetail) { + private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) { Measure measure = new Measure(); measure.setExpr("1"); if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java index 6737381b6..737841f41 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Datasource.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.semantic.model.domain.pojo; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; @@ -13,7 +13,7 @@ public class Datasource extends SchemaItem { private Long databaseId; - private DatasourceDetail datasourceDetail; + private ModelDetail datasourceDetail; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java index a085da2bb..0ed3c9e35 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java @@ -27,8 +27,6 @@ public class MetaFilter { private List ids; - private Long datasourceId; - public MetaFilter(List modelIds) { this.modelIds = modelIds; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/ModelFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/ModelFilter.java new file mode 100644 index 000000000..172942925 --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/ModelFilter.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.semantic.model.domain.pojo; + +import lombok.Data; + +import java.util.List; + +@Data +public class ModelFilter extends MetaFilter { + + private Long databaseId; + + private List domainIds; + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java deleted file mode 100644 index 148bddee2..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.repository; - - -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; - -import java.util.List; - -public interface DatasourceRepository { - - void createDatasource(DatasourceDO datasourceDO); - - void updateDatasource(DatasourceDO datasourceDO); - - List getDatasourceList(); - - List getDatasourceList(Long modelId); - - List getDatasourceByDatabase(Long databaseId); - - DatasourceDO getDatasourceById(Long id); - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/ModelRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/ModelRepository.java index 3712403e5..21ef1a134 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/ModelRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/ModelRepository.java @@ -1,6 +1,9 @@ package com.tencent.supersonic.semantic.model.domain.repository; + import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; + import java.util.List; public interface ModelRepository { @@ -9,10 +12,9 @@ public interface ModelRepository { void updateModel(ModelDO modelDO); - void deleteModel(Long id); - - List getModelList(); + List getModelList(ModelFilter modelFilter); ModelDO getModelById(Long id); + void batchUpdate(List modelDOS); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java deleted file mode 100644 index 4f95ca866..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java +++ /dev/null @@ -1,201 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.utils; - -import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.semantic.api.model.enums.MetricTypeEnum; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; -import com.tencent.supersonic.semantic.api.model.pojo.Dim; -import com.tencent.supersonic.semantic.api.model.pojo.Identify; -import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; -import com.tencent.supersonic.semantic.api.model.request.DimensionReq; -import com.tencent.supersonic.semantic.api.model.request.MetricReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; -import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; -import java.util.Date; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - - -public class DatasourceConverter { - - - public static DatasourceDO convert(DatasourceReq datasourceReq, User user) { - DatasourceDO datasource = new DatasourceDO(); - DatasourceDetail datasourceDetail = getDatasourceDetail(datasourceReq); - datasourceReq.createdBy(user.getName()); - BeanMapper.mapper(datasourceReq, datasource); - datasource.setStatus(StatusEnum.ONLINE.getCode()); - datasource.setDatasourceDetail(JSONObject.toJSONString(datasourceDetail)); - return datasource; - } - - - public static DatasourceDO convert(DatasourceDO datasourceDO, DatasourceReq datasourceReq, User user) { - DatasourceDetail datasourceDetail = getDatasourceDetail(datasourceReq); - BeanMapper.mapper(datasourceReq, datasourceDO); - datasourceDO.setDatasourceDetail(JSONObject.toJSONString((datasourceDetail))); - datasourceDO.setUpdatedBy(user.getName()); - datasourceDO.setUpdatedAt(new Date()); - return datasourceDO; - } - - public static DatasourceResp convert(DatasourceDO datasourceDO) { - DatasourceResp datasourceResp = new DatasourceResp(); - BeanUtils.copyProperties(datasourceDO, datasourceResp); - datasourceResp.setDatasourceDetail( - JSONObject.parseObject(datasourceDO.getDatasourceDetail(), DatasourceDetail.class)); - return datasourceResp; - } - - public static MeasureResp convert(Measure measure, DatasourceResp datasourceResp) { - MeasureResp measureResp = new MeasureResp(); - BeanUtils.copyProperties(measure, measureResp); - measureResp.setDatasourceId(datasourceResp.getId()); - measureResp.setDatasourceName(datasourceResp.getName()); - measureResp.setDatasourceBizName(datasourceResp.getBizName()); - measureResp.setModelId(datasourceResp.getModelId()); - return measureResp; - } - - public static DimensionReq convert(Dim dim, DatasourceDO datasourceDO) { - DimensionReq dimensionReq = new DimensionReq(); - dimensionReq.setName(dim.getName()); - dimensionReq.setBizName(dim.getBizName()); - dimensionReq.setDescription(dim.getName()); - dimensionReq.setSemanticType("CATEGORY"); - dimensionReq.setDatasourceId(datasourceDO.getId()); - dimensionReq.setModelId(datasourceDO.getModelId()); - dimensionReq.setExpr(dim.getBizName()); - dimensionReq.setType("categorical"); - dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); - dimensionReq.setIsTag(dim.getIsTag()); - return dimensionReq; - } - - public static MetricReq convert(Measure measure, DatasourceDO datasourceDO) { - measure.setDatasourceId(datasourceDO.getId()); - MetricReq metricReq = new MetricReq(); - metricReq.setName(measure.getName()); - metricReq.setBizName(measure.getBizName().replaceFirst(datasourceDO.getBizName() + "_", "")); - metricReq.setDescription(measure.getName()); - metricReq.setModelId(datasourceDO.getModelId()); - metricReq.setMetricType(MetricTypeEnum.ATOMIC); - MetricTypeParams exprTypeParams = new MetricTypeParams(); - exprTypeParams.setExpr(measure.getBizName()); - exprTypeParams.setMeasures(Lists.newArrayList(measure)); - metricReq.setTypeParams(exprTypeParams); - return metricReq; - } - - public static DimensionReq convert(Identify identify, DatasourceDO datasourceDO) { - DimensionReq dimensionReq = new DimensionReq(); - dimensionReq.setName(identify.getName()); - dimensionReq.setBizName(identify.getBizName()); - dimensionReq.setDescription(identify.getName()); - dimensionReq.setSemanticType("CATEGORY"); - dimensionReq.setDatasourceId(datasourceDO.getId()); - dimensionReq.setModelId(datasourceDO.getModelId()); - dimensionReq.setExpr(identify.getBizName()); - dimensionReq.setType(identify.getType()); - return dimensionReq; - } - - public static List convertList(List datasourceDOS) { - List datasourceDescs = Lists.newArrayList(); - if (!CollectionUtils.isEmpty(datasourceDOS)) { - datasourceDescs = datasourceDOS.stream().map(DatasourceConverter::convert).collect(Collectors.toList()); - } - return datasourceDescs; - } - - - private static boolean isCreateDimension(Dim dim) { - return dim.getIsCreateDimension() == 1 - && StringUtils.isNotBlank(dim.getName()) - && !dim.getType().equalsIgnoreCase("time"); - } - - private static boolean isCreateMetric(Measure measure) { - return measure.getIsCreateMetric() == 1 - && StringUtils.isNotBlank(measure.getName()); - } - - public static List getDimToCreateDimension(DatasourceDetail datasourceDetail) { - return datasourceDetail.getDimensions().stream() - .filter(DatasourceConverter::isCreateDimension) - .collect(Collectors.toList()); - } - - public static List getMeasureToCreateMetric(DatasourceDetail datasourceDetail) { - return datasourceDetail.getMeasures().stream() - .filter(DatasourceConverter::isCreateMetric) - .collect(Collectors.toList()); - } - - public static List convertDimensionList(DatasourceDO datasourceDO) { - List dimensionReqs = Lists.newArrayList(); - DatasourceDetail datasourceDetail = JSONObject.parseObject(datasourceDO.getDatasourceDetail(), - DatasourceDetail.class); - List dims = getDimToCreateDimension(datasourceDetail); - if (!CollectionUtils.isEmpty(dims)) { - dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName())) - .map(dim -> convert(dim, datasourceDO)).collect(Collectors.toList()); - } - List identifies = datasourceDetail.getIdentifiers(); - if (CollectionUtils.isEmpty(identifies)) { - return dimensionReqs; - } - dimensionReqs.addAll(identifies.stream() - .filter(i -> i.getType().equalsIgnoreCase("primary")) - .filter(i -> StringUtils.isNotBlank(i.getName())) - .map(identify -> convert(identify, datasourceDO)).collect(Collectors.toList())); - return dimensionReqs; - } - - - public static List convertMetricList(DatasourceDO datasourceDO) { - DatasourceDetail datasourceDetail = JSONObject.parseObject(datasourceDO.getDatasourceDetail(), - DatasourceDetail.class); - List measures = getMeasureToCreateMetric(datasourceDetail); - if (CollectionUtils.isEmpty(measures)) { - return Lists.newArrayList(); - } - return measures.stream().map(measure -> convert(measure, datasourceDO)).collect(Collectors.toList()); - } - - public static Datasource datasourceInfo2Datasource(DatasourceResp datasourceResp) { - Datasource datasource = new Datasource(); - BeanUtils.copyProperties(datasourceResp, datasource); - return datasource; - } - - private static DatasourceDetail getDatasourceDetail(DatasourceReq datasourceReq) { - DatasourceDetail datasourceDetail = new DatasourceDetail(); - BeanMapper.mapper(datasourceReq, datasourceDetail); - List measures = datasourceDetail.getMeasures(); - for (Measure measure : measures) { - if (StringUtils.isBlank(measure.getExpr())) { - measure.setExpr(measure.getBizName()); - } - if (StringUtils.isBlank(measure.getConstraint())) { - measure.setConstraint(null); - } - if (StringUtils.isBlank(measure.getAlias())) { - measure.setAlias(null); - } - measure.setBizName(String.format("%s_%s", datasourceReq.getBizName(), measure.getBizName())); - } - return datasourceDetail; - } -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java index c6e30be3e..158555b27 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java @@ -3,25 +3,24 @@ package com.tencent.supersonic.semantic.model.domain.utils; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; -import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; import com.tencent.supersonic.semantic.model.domain.pojo.Dimension; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; - public class DimensionConverter { public static DimensionDO convert(DimensionDO dimensionDO, DimensionReq dimensionReq) { @@ -57,22 +56,18 @@ public class DimensionConverter { } public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO, - Map fullPathMap, - Map datasourceRespMap) { + Map modelRespMap) { DimensionResp dimensionResp = new DimensionResp(); BeanUtils.copyProperties(dimensionDO, dimensionResp); - dimensionResp.setFullPath(fullPathMap.get(dimensionDO.getModelId()) + "/" + dimensionDO.getBizName()); - dimensionResp.setDatasourceId( - datasourceRespMap.getOrDefault(dimensionResp.getDatasourceId(), new DatasourceResp()).getId()); - dimensionResp.setDatasourceName( - datasourceRespMap.getOrDefault(dimensionResp.getDatasourceId(), new DatasourceResp()).getName()); - dimensionResp.setDatasourceBizName( - datasourceRespMap.getOrDefault(dimensionResp.getDatasourceId(), new DatasourceResp()).getBizName()); + dimensionResp.setModelName( + modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getName()); + dimensionResp.setModelBizName( + modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getBizName()); if (dimensionDO.getDefaultValues() != null) { dimensionResp.setDefaultValues(JSONObject.parseObject(dimensionDO.getDefaultValues(), List.class)); } - dimensionResp.setDatasourceFilterSql( - datasourceRespMap.getOrDefault(dimensionResp.getDatasourceId(), new DatasourceResp()).getFilterSql()); + dimensionResp.setModelFilterSql( + modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getFilterSql()); if (Strings.isNotEmpty(dimensionDO.getDimValueMaps())) { dimensionResp.setDimValueMaps(JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class)); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DomainConvert.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DomainConvert.java index e453e5f6e..355ea9cda 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DomainConvert.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DomainConvert.java @@ -6,39 +6,25 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.request.DomainReq; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDO; -import com.tencent.supersonic.semantic.model.domain.pojo.Domain; - -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + public class DomainConvert { - public static Domain convert(DomainReq domainReq) { - Domain domain = new Domain(); - BeanUtils.copyProperties(domainReq, domain); - domain.setStatus(StatusEnum.ONLINE.getCode()); - return domain; - } - - public static DomainDO convert(Domain domain, User user) { + public static DomainDO convert(DomainReq domainReq, User user) { DomainDO domainDO = new DomainDO(); - BeanUtils.copyProperties(domain, domainDO); - domainDO.setCreatedBy(user.getName()); - domainDO.setUpdatedBy(user.getName()); - domainDO.setCreatedAt(new Date()); - domainDO.setUpdatedAt(new Date()); - domainDO.setAdmin(String.join(",", domain.getAdmins())); - domainDO.setAdminOrg(String.join(",", domain.getAdminOrgs())); - domainDO.setViewer(String.join(",", domain.getViewers())); - domainDO.setViewOrg(String.join(",", domain.getViewOrgs())); + domainReq.createdBy(user.getName()); + BeanUtils.copyProperties(domainReq, domainDO); + domainDO.setAdmin(String.join(",", domainReq.getAdmins())); + domainDO.setAdminOrg(String.join(",", domainReq.getAdminOrgs())); + domainDO.setViewer(String.join(",", domainReq.getViewers())); + domainDO.setViewOrg(String.join(",", domainReq.getViewOrgs())); return domainDO; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java index b98885bbe..dd8f14224 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java @@ -4,25 +4,25 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; +import com.tencent.supersonic.semantic.api.model.request.MetricReq; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; -import com.tencent.supersonic.semantic.api.model.request.MetricReq; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; import com.tencent.supersonic.semantic.model.domain.pojo.Metric; +import org.springframework.beans.BeanUtils; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.springframework.beans.BeanUtils; - public class MetricConverter { public static MetricDO convert2MetricDO(MetricReq metricReq) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java deleted file mode 100644 index 1e9267372..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.tencent.supersonic.semantic.model.domain.utils; - - -import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; -import com.tencent.supersonic.semantic.api.model.pojo.Entity; -import com.tencent.supersonic.semantic.api.model.request.ModelReq; -import com.tencent.supersonic.semantic.api.model.response.DomainResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import java.util.Arrays; -import java.util.Map; - -public class ModelConvert { - - public static ModelDO convert(ModelReq modelReq) { - ModelDO modelDO = new ModelDO(); - BeanMapper.mapper(modelReq, modelDO); - modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); - modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions())); - modelDO.setStatus(StatusEnum.ONLINE.getCode()); - return modelDO; - } - - public static ModelResp convert(ModelDO modelDO) { - ModelResp modelResp = new ModelResp(); - BeanUtils.copyProperties(modelDO, modelResp); - modelResp.setAdmins(StringUtils.isBlank(modelDO.getAdmin()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getAdmin().split(","))); - modelResp.setAdminOrgs(StringUtils.isBlank(modelDO.getAdminOrg()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getAdminOrg().split(","))); - modelResp.setViewers(StringUtils.isBlank(modelDO.getViewer()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getViewer().split(","))); - modelResp.setViewOrgs(StringUtils.isBlank(modelDO.getViewOrg()) - ? Lists.newArrayList() : Arrays.asList(modelDO.getViewOrg().split(","))); - modelResp.setEntity(JsonUtil.toObject(modelDO.getEntity(), Entity.class)); - modelResp.setDrillDownDimensions(JsonUtil.toList(modelDO.getDrillDownDimensions(), DrillDownDimension.class)); - return modelResp; - } - - public static ModelResp convert(ModelDO modelDO, - Map domainRespMap) { - ModelResp modelResp = convert(modelDO); - DomainResp domainResp = domainRespMap.get(modelResp.getDomainId()); - if (domainResp != null) { - String fullBizNamePath = domainResp.getFullPath() + modelResp.getBizName(); - modelResp.setFullPath(fullBizNamePath); - } - return modelResp; - } - - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java new file mode 100644 index 000000000..90ae94fd3 --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java @@ -0,0 +1,218 @@ +package com.tencent.supersonic.semantic.model.domain.utils; + +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.semantic.api.model.enums.MetricTypeEnum; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; +import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; +import com.tencent.supersonic.semantic.api.model.pojo.Measure; +import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; +import com.tencent.supersonic.semantic.api.model.request.DimensionReq; +import com.tencent.supersonic.semantic.api.model.request.MetricReq; +import com.tencent.supersonic.semantic.api.model.request.ModelReq; +import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import com.tencent.supersonic.semantic.api.model.response.MeasureResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + + +public class ModelConverter { + + + public static ModelDO convert(ModelReq modelReq, User user) { + ModelDO modelDO = new ModelDO(); + ModelDetail modelDetail = getModelDetail(modelReq); + modelReq.createdBy(user.getName()); + BeanMapper.mapper(modelReq, modelDO); + modelDO.setStatus(StatusEnum.ONLINE.getCode()); + modelDO.setModelDetail(JSONObject.toJSONString(modelDetail)); + modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions())); + return modelDO; + } + + public static ModelResp convert(ModelDO modelDO) { + ModelResp modelResp = new ModelResp(); + BeanUtils.copyProperties(modelDO, modelResp); + modelResp.setAdmins(StringUtils.isBlank(modelDO.getAdmin()) + ? Lists.newArrayList() : Arrays.asList(modelDO.getAdmin().split(","))); + modelResp.setAdminOrgs(StringUtils.isBlank(modelDO.getAdminOrg()) + ? Lists.newArrayList() : Arrays.asList(modelDO.getAdminOrg().split(","))); + modelResp.setViewers(StringUtils.isBlank(modelDO.getViewer()) + ? Lists.newArrayList() : Arrays.asList(modelDO.getViewer().split(","))); + modelResp.setViewOrgs(StringUtils.isBlank(modelDO.getViewOrg()) + ? Lists.newArrayList() : Arrays.asList(modelDO.getViewOrg().split(","))); + modelResp.setDrillDownDimensions(JsonUtil.toList(modelDO.getDrillDownDimensions(), DrillDownDimension.class)); + modelResp.setModelDetail(JSONObject.parseObject(modelDO.getModelDetail(), ModelDetail.class)); + return modelResp; + } + + public static ModelResp convert(ModelDO modelDO, Map domainRespMap) { + ModelResp modelResp = convert(modelDO); + DomainResp domainResp = domainRespMap.get(modelResp.getDomainId()); + if (domainResp != null) { + String fullBizNamePath = domainResp.getFullPath() + modelResp.getBizName(); + modelResp.setFullPath(fullBizNamePath); + } + return modelResp; + } + + + + public static ModelDO convert(ModelDO modelDO, ModelReq modelReq, User user) { + ModelDetail modelDetail = getModelDetail(modelReq); + BeanMapper.mapper(modelReq, modelDO); + modelDO.setModelDetail(JSONObject.toJSONString((modelDetail))); + modelDO.setUpdatedBy(user.getName()); + modelDO.setUpdatedAt(new Date()); + return modelDO; + } + + public static MeasureResp convert(Measure measure, ModelResp modelResp) { + MeasureResp measureResp = new MeasureResp(); + BeanUtils.copyProperties(measure, measureResp); + measureResp.setDatasourceId(modelResp.getId()); + measureResp.setDatasourceName(modelResp.getName()); + measureResp.setDatasourceBizName(modelResp.getBizName()); + measureResp.setModelId(modelResp.getId()); + return measureResp; + } + + public static DimensionReq convert(Dim dim, ModelDO modelDO) { + DimensionReq dimensionReq = new DimensionReq(); + dimensionReq.setName(dim.getName()); + dimensionReq.setBizName(dim.getBizName()); + dimensionReq.setDescription(dim.getName()); + dimensionReq.setSemanticType("CATEGORY"); + dimensionReq.setModelId(modelDO.getId()); + dimensionReq.setExpr(dim.getBizName()); + dimensionReq.setType("categorical"); + dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); + dimensionReq.setIsTag(dim.getIsTag()); + return dimensionReq; + } + + public static MetricReq convert(Measure measure, ModelDO modelDO) { + measure.setDatasourceId(modelDO.getId()); + MetricReq metricReq = new MetricReq(); + metricReq.setName(measure.getName()); + metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", "")); + metricReq.setDescription(measure.getName()); + metricReq.setModelId(modelDO.getId()); + metricReq.setMetricType(MetricTypeEnum.ATOMIC); + MetricTypeParams exprTypeParams = new MetricTypeParams(); + exprTypeParams.setExpr(measure.getBizName()); + exprTypeParams.setMeasures(Lists.newArrayList(measure)); + metricReq.setTypeParams(exprTypeParams); + return metricReq; + } + + public static DimensionReq convert(Identify identify, ModelDO modelDO) { + DimensionReq dimensionReq = new DimensionReq(); + dimensionReq.setName(identify.getName()); + dimensionReq.setBizName(identify.getBizName()); + dimensionReq.setDescription(identify.getName()); + dimensionReq.setSemanticType("CATEGORY"); + dimensionReq.setModelId(modelDO.getId()); + dimensionReq.setExpr(identify.getBizName()); + dimensionReq.setType(identify.getType()); + return dimensionReq; + } + + public static List convertList(List modelDOS) { + List modelDescs = Lists.newArrayList(); + if (!CollectionUtils.isEmpty(modelDOS)) { + modelDescs = modelDOS.stream().map(ModelConverter::convert).collect(Collectors.toList()); + } + return modelDescs; + } + + + private static boolean isCreateDimension(Dim dim) { + return dim.getIsCreateDimension() == 1 + && StringUtils.isNotBlank(dim.getName()) + && !dim.getType().equalsIgnoreCase("time"); + } + + private static boolean isCreateMetric(Measure measure) { + return measure.getIsCreateMetric() == 1 + && StringUtils.isNotBlank(measure.getName()); + } + + public static List getDimToCreateDimension(ModelDetail modelDetail) { + return modelDetail.getDimensions().stream() + .filter(ModelConverter::isCreateDimension) + .collect(Collectors.toList()); + } + + public static List getMeasureToCreateMetric(ModelDetail modelDetail) { + return modelDetail.getMeasures().stream() + .filter(ModelConverter::isCreateMetric) + .collect(Collectors.toList()); + } + + public static List convertDimensionList(ModelDO modelDO) { + List dimensionReqs = Lists.newArrayList(); + ModelDetail modelDetail = JSONObject.parseObject(modelDO.getModelDetail(), + ModelDetail.class); + List dims = getDimToCreateDimension(modelDetail); + if (!CollectionUtils.isEmpty(dims)) { + dimensionReqs = dims.stream().filter(dim -> StringUtils.isNotBlank(dim.getName())) + .map(dim -> convert(dim, modelDO)).collect(Collectors.toList()); + } + List identifies = modelDetail.getIdentifiers(); + if (CollectionUtils.isEmpty(identifies)) { + return dimensionReqs; + } + dimensionReqs.addAll(identifies.stream() + .filter(i -> i.getType().equalsIgnoreCase("primary")) + .filter(i -> StringUtils.isNotBlank(i.getName())) + .map(identify -> convert(identify, modelDO)).collect(Collectors.toList())); + return dimensionReqs; + } + + + public static List convertMetricList(ModelDO modelDO) { + ModelDetail modelDetail = JSONObject.parseObject(modelDO.getModelDetail(), + ModelDetail.class); + List measures = getMeasureToCreateMetric(modelDetail); + if (CollectionUtils.isEmpty(measures)) { + return Lists.newArrayList(); + } + return measures.stream().map(measure -> convert(measure, modelDO)).collect(Collectors.toList()); + } + + private static ModelDetail getModelDetail(ModelReq modelReq) { + ModelDetail modelDetail = new ModelDetail(); + BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); + List measures = modelDetail.getMeasures(); + for (Measure measure : measures) { + if (StringUtils.isBlank(measure.getExpr())) { + measure.setExpr(measure.getBizName()); + } + if (StringUtils.isBlank(measure.getConstraint())) { + measure.setConstraint(null); + } + if (StringUtils.isBlank(measure.getAlias())) { + measure.setAlias(null); + } + measure.setBizName(String.format("%s_%s", modelReq.getBizName(), measure.getBizName())); + } + return modelDetail; + } +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DomainDOMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DomainDOMapper.java index e3cc8be39..2db653f95 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DomainDOMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DomainDOMapper.java @@ -1,58 +1,10 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDOExample; import org.apache.ibatis.annotations.Mapper; -import java.util.List; - @Mapper -public interface DomainDOMapper { - /** - * - * @mbg.generated - */ - long countByExample(DomainDOExample example); +public interface DomainDOMapper extends BaseMapper { - /** - * - * @mbg.generated - */ - int deleteByPrimaryKey(Long id); - - /** - * - * @mbg.generated - */ - int insert(DomainDO record); - - /** - * - * @mbg.generated - */ - int insertSelective(DomainDO record); - - /** - * - * @mbg.generated - */ - List selectByExample(DomainDOExample example); - - /** - * - * @mbg.generated - */ - DomainDO selectByPrimaryKey(Long id); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKeySelective(DomainDO record); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKey(DomainDO record); } \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOCustomMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOCustomMapper.java new file mode 100644 index 000000000..a5df4b5be --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOCustomMapper.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.semantic.model.infrastructure.mapper; + + +import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; +import org.apache.ibatis.annotations.Mapper; +import java.util.List; + +@Mapper +public interface ModelDOCustomMapper { + + void batchUpdateStatus(List modelDOS); + + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOMapper.java index 9395e78a8..201f49337 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelDOMapper.java @@ -1,89 +1,11 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDOExample; -import java.util.List; - import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; @Mapper -public interface ModelDOMapper { - /** - * - * @mbg.generated - */ - long countByExample(ModelDOExample example); +public interface ModelDOMapper extends BaseMapper { - /** - * - * @mbg.generated - */ - int deleteByPrimaryKey(Long id); - /** - * - * @mbg.generated - */ - int insert(ModelDO record); - - /** - * - * @mbg.generated - */ - int insertSelective(ModelDO record); - - /** - * - * @mbg.generated - */ - List selectByExampleWithBLOBs(ModelDOExample example); - - /** - * - * @mbg.generated - */ - List selectByExample(ModelDOExample example); - - /** - * - * @mbg.generated - */ - ModelDO selectByPrimaryKey(Long id); - - /** - * - * @mbg.generated - */ - int updateByExampleSelective(@Param("record") ModelDO record, @Param("example") ModelDOExample example); - - /** - * - * @mbg.generated - */ - int updateByExampleWithBLOBs(@Param("record") ModelDO record, @Param("example") ModelDOExample example); - - /** - * - * @mbg.generated - */ - int updateByExample(@Param("record") ModelDO record, @Param("example") ModelDOExample example); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKeySelective(ModelDO record); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKeyWithBLOBs(ModelDO record); - - /** - * - * @mbg.generated - */ - int updateByPrimaryKey(ModelDO record); } \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DatasourceDOMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelRelaDOMapper.java similarity index 54% rename from semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DatasourceDOMapper.java rename to semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelRelaDOMapper.java index eebe29ba9..1690e7598 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DatasourceDOMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/ModelRelaDOMapper.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper; + import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; +import com.tencent.supersonic.semantic.model.domain.dataobject.ModelRelaDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface DatasourceDOMapper extends BaseMapper { +public interface ModelRelaDOMapper extends BaseMapper { - -} \ No newline at end of file +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java deleted file mode 100644 index 3c6c55c5c..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.tencent.supersonic.semantic.model.infrastructure.repository; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; -import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository; -import com.tencent.supersonic.semantic.model.infrastructure.mapper.DatasourceDOMapper; - -import java.util.List; -import org.springframework.stereotype.Component; - - -@Component -public class DatasourceRepositoryImpl implements DatasourceRepository { - - - private DatasourceDOMapper datasourceMapper; - - public DatasourceRepositoryImpl(DatasourceDOMapper datasourceMapper) { - this.datasourceMapper = datasourceMapper; - } - - - @Override - public void createDatasource(DatasourceDO datasourceDO) { - datasourceMapper.insert(datasourceDO); - } - - @Override - public void updateDatasource(DatasourceDO datasourceDO) { - datasourceMapper.updateById(datasourceDO); - } - - @Override - public List getDatasourceList() { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.lambda().ne(DatasourceDO::getStatus, StatusEnum.DELETED.getCode()); - return datasourceMapper.selectList(wrapper); - } - - @Override - public List getDatasourceList(Long modelId) { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.lambda().ne(DatasourceDO::getStatus, StatusEnum.DELETED.getCode()) - .eq(DatasourceDO::getModelId, modelId); - return datasourceMapper.selectList(wrapper); - } - - @Override - public List getDatasourceByDatabase(Long databaseId) { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.lambda().ne(DatasourceDO::getStatus, StatusEnum.DELETED.getCode()) - .eq(DatasourceDO::getDatabaseId, databaseId); - return datasourceMapper.selectList(wrapper); - } - - @Override - public DatasourceDO getDatasourceById(Long id) { - return datasourceMapper.selectById(id); - } - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DomainRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DomainRepositoryImpl.java index 8b6a475f0..40885d68e 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DomainRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DomainRepositoryImpl.java @@ -1,14 +1,14 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDOExample; import com.tencent.supersonic.semantic.model.domain.repository.DomainRepository; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DomainDOMapper; -import java.util.List; - import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.List; + @Component @Slf4j @@ -27,23 +27,22 @@ public class DomainRepositoryImpl implements DomainRepository { @Override public void updateDomain(DomainDO metaDomainDO) { - domainDOMapper.updateByPrimaryKey(metaDomainDO); + domainDOMapper.updateById(metaDomainDO); } @Override public void deleteDomain(Long id) { - domainDOMapper.deleteByPrimaryKey(id); + domainDOMapper.deleteById(id); } @Override public List getDomainList() { - DomainDOExample metaDomainDOExample = new DomainDOExample(); - return domainDOMapper.selectByExample(metaDomainDOExample); + return domainDOMapper.selectList(Wrappers.emptyWrapper()); } @Override public DomainDO getDomainById(Long id) { - return domainDOMapper.selectByPrimaryKey(id); + return domainDOMapper.selectById(id); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java index 39d6986ed..2609ada3d 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java @@ -1,49 +1,67 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDOExample; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; +import com.tencent.supersonic.semantic.model.infrastructure.mapper.ModelDOCustomMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.ModelDOMapper; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import java.util.List; + @Component public class ModelRepositoryImpl implements ModelRepository { + private ModelDOMapper modelDOMapper; - public ModelRepositoryImpl(ModelDOMapper modelDOMapper) { + private ModelDOCustomMapper modelDOCustomMapper; + + public ModelRepositoryImpl(ModelDOMapper modelDOMapper, + ModelDOCustomMapper modelDOCustomMapper) { this.modelDOMapper = modelDOMapper; + this.modelDOCustomMapper = modelDOCustomMapper; + } + + + @Override + public void createModel(ModelDO datasourceDO) { + modelDOMapper.insert(datasourceDO); } @Override - public void createModel(ModelDO modelDO) { - modelDOMapper.insert(modelDO); + public void updateModel(ModelDO datasourceDO) { + modelDOMapper.updateById(datasourceDO); } @Override - public void updateModel(ModelDO modelDO) { - modelDOMapper.updateByPrimaryKeySelective(modelDO); - } - - @Override - public void deleteModel(Long id) { - ModelDO modelDO = modelDOMapper.selectByPrimaryKey(id); - modelDO.setStatus(StatusEnum.DELETED.getCode()); - modelDOMapper.updateByPrimaryKey(modelDO); - } - - @Override - public List getModelList() { - ModelDOExample modelDOExample = new ModelDOExample(); - modelDOExample.createCriteria().andStatusNotEqualTo(StatusEnum.DELETED.getCode()); - return modelDOMapper.selectByExampleWithBLOBs(modelDOExample); + public List getModelList(ModelFilter modelFilter) { + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda().ne(ModelDO::getStatus, StatusEnum.DELETED.getCode()); + if (modelFilter.getDatabaseId() != null) { + wrapper.lambda().eq(ModelDO::getDatabaseId, modelFilter.getDatabaseId()); + } + if (!CollectionUtils.isEmpty(modelFilter.getDomainIds())) { + wrapper.lambda().in(ModelDO::getDomainId, modelFilter.getDomainIds()); + } + if (!CollectionUtils.isEmpty(modelFilter.getIds())) { + wrapper.lambda().in(ModelDO::getId, modelFilter.getIds()); + } + return modelDOMapper.selectList(wrapper); } @Override public ModelDO getModelById(Long id) { - return modelDOMapper.selectByPrimaryKey(id); + return modelDOMapper.selectById(id); } + + @Override + public void batchUpdate(List modelDOS) { + modelDOCustomMapper.batchUpdateStatus(modelDOS); + } + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java deleted file mode 100644 index 291d3709d..000000000 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.tencent.supersonic.semantic.model.rest; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.semantic.api.model.response.MeasureResp; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - - -@RestController -@RequestMapping("/api/semantic/datasource") -public class DatasourceController { - - private DatasourceService datasourceService; - - public DatasourceController(DatasourceService datasourceService) { - this.datasourceService = datasourceService; - } - - @PostMapping("/createDatasource") - public DatasourceResp createDatasource(@RequestBody DatasourceReq datasourceReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { - User user = UserHolder.findUser(request, response); - return datasourceService.createDatasource(datasourceReq, user); - } - - @PostMapping("/updateDatasource") - public DatasourceResp updateDatasource(@RequestBody DatasourceReq datasourceReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { - User user = UserHolder.findUser(request, response); - return datasourceService.updateDatasource(datasourceReq, user); - } - - @GetMapping("/getDatasourceList/{modelId}") - public List getDatasourceList(@PathVariable("modelId") Long modelId) { - return datasourceService.getDatasourceListNoMeasurePrefix(modelId); - } - - @GetMapping("/getMeasureListOfModel/{modelId}") - public List getMeasureListOfModel(@PathVariable("modelId") Long modelId) { - return datasourceService.getMeasureListOfModel(Lists.newArrayList(modelId)); - } - - @DeleteMapping("deleteDatasource/{id}") - public boolean deleteDatasource(@PathVariable("id") Long id, - HttpServletRequest request, HttpServletResponse response) { - User user = UserHolder.findUser(request, response); - datasourceService.deleteDatasource(id, user); - return true; - } - -} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java index a96c142a1..5a8332e09 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java @@ -3,10 +3,12 @@ package com.tencent.supersonic.semantic.model.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.PostMapping; @@ -33,8 +35,7 @@ public class ModelController { @PostMapping("/createModel") public Boolean createModel(@RequestBody ModelReq modelReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); modelService.createModel(modelReq, user); return true; @@ -42,8 +43,7 @@ public class ModelController { @PostMapping("/updateModel") public Boolean updateModel(@RequestBody ModelReq modelReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); modelService.updateModel(modelReq, user); return true; @@ -51,8 +51,7 @@ public class ModelController { @DeleteMapping("/deleteModel/{modelId}") public Boolean deleteModel(@PathVariable("modelId") Long modelId, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); modelService.deleteModel(modelId, user); return true; @@ -60,8 +59,7 @@ public class ModelController { @GetMapping("/getModelList/{domainId}") public List getModelList(@PathVariable("domainId") Long domainId, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN); } @@ -74,8 +72,10 @@ public class ModelController { @GetMapping("/getModelListByIds/{modelIds}") public List getModelListByIds(@PathVariable("modelIds") String modelIds) { - return modelService.getModelList(Arrays.stream(modelIds.split(",")).map(Long::parseLong) - .collect(Collectors.toList())); + List ids = Arrays.stream(modelIds.split(",")).map(Long::parseLong).collect(Collectors.toList()); + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setIds(ids); + return modelService.getModelList(modelFilter); } @GetMapping("/getModelDatabase/{modelId}") @@ -83,4 +83,14 @@ public class ModelController { return modelService.getDatabaseByModelId(modelId); } + + @PostMapping("/batchUpdateStatus") + public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + modelService.batchUpdateStatus(metaBatchReq, user); + return true; + } + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelRelaController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelRelaController.java new file mode 100644 index 000000000..aee051d8a --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelRelaController.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.semantic.model.rest; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/api/semantic/modelRela") +public class ModelRelaController { + + @Autowired + private ModelRelaService modelRelaService; + + @PostMapping + public boolean save(@RequestBody ModelRela modelRela, User user) { + modelRelaService.save(modelRela, user); + return true; + } + + @PutMapping + public boolean update(@RequestBody ModelRela modelRela, User user) { + modelRelaService.update(modelRela, user); + return true; + } + + @RequestMapping("/list") + public List getModelRelaList(@RequestParam("domainId") Long domainId) { + return modelRelaService.getModelRelaList(domainId); + } + + @DeleteMapping("/{id}") + public void delete(@PathVariable("id") Long id) { + modelRelaService.delete(id); + } + + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java index 2e9e9555f..69ebb52fd 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ViewInfoController.java @@ -5,12 +5,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.semantic.api.model.request.ViewInfoReq; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaRelaResp; -import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; import com.tencent.supersonic.semantic.model.application.ViewInfoServiceImpl; - -import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -19,6 +15,10 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; + @RestController @RequestMapping("/api/semantic/viewInfo") public class ViewInfoController { @@ -47,9 +47,9 @@ public class ViewInfoController { viewInfoServiceImpl.deleteViewInfo(id); } - @GetMapping("/getDomainSchemaRela/{modelId}") - public List getDomainSchema(@PathVariable("modelId") Long modelId) { - return viewInfoServiceImpl.getDomainSchema(modelId); + @GetMapping("/getDomainSchemaRela/{domainId}") + public List getDomainSchema(@PathVariable("domainId") Long domainId) { + return viewInfoServiceImpl.getDomainSchema(domainId); } diff --git a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml b/semantic/model/src/main/resources/mapper/ModelDOMapper.xml deleted file mode 100644 index 486c51018..000000000 --- a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml +++ /dev/null @@ -1,480 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, name, biz_name, domain_id, alias, description, viewer, view_org, admin, admin_org, - is_open, created_by, created_at, updated_by, updated_at, drill_down_dimensions, status - - - entity - - - - - - delete from s2_model - where id = #{id,jdbcType=BIGINT} - - - insert into s2_model (id, name, biz_name, - domain_id, alias, description, - viewer, view_org, admin, - admin_org, is_open, created_by, - created_at, updated_by, updated_at, - drill_down_dimensions, status, entity - ) - values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR}, - #{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR}, - #{viewer,jdbcType=VARCHAR}, #{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR}, - #{adminOrg,jdbcType=VARCHAR}, #{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR}, - #{createdAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, - #{drillDownDimensions,jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, #{entity,jdbcType=LONGVARCHAR} - ) - - - insert into s2_model - - - id, - - - name, - - - biz_name, - - - domain_id, - - - alias, - - - description, - - - viewer, - - - view_org, - - - admin, - - - admin_org, - - - is_open, - - - created_by, - - - created_at, - - - updated_by, - - - updated_at, - - - drill_down_dimensions, - - - status, - - - entity, - - - - - #{id,jdbcType=BIGINT}, - - - #{name,jdbcType=VARCHAR}, - - - #{bizName,jdbcType=VARCHAR}, - - - #{domainId,jdbcType=BIGINT}, - - - #{alias,jdbcType=VARCHAR}, - - - #{description,jdbcType=VARCHAR}, - - - #{viewer,jdbcType=VARCHAR}, - - - #{viewOrg,jdbcType=VARCHAR}, - - - #{admin,jdbcType=VARCHAR}, - - - #{adminOrg,jdbcType=VARCHAR}, - - - #{isOpen,jdbcType=INTEGER}, - - - #{createdBy,jdbcType=VARCHAR}, - - - #{createdAt,jdbcType=TIMESTAMP}, - - - #{updatedBy,jdbcType=VARCHAR}, - - - #{updatedAt,jdbcType=TIMESTAMP}, - - - #{drillDownDimensions,jdbcType=VARCHAR}, - - - #{status,jdbcType=INTEGER}, - - - #{entity,jdbcType=LONGVARCHAR}, - - - - - - update s2_model - - - id = #{record.id,jdbcType=BIGINT}, - - - name = #{record.name,jdbcType=VARCHAR}, - - - biz_name = #{record.bizName,jdbcType=VARCHAR}, - - - domain_id = #{record.domainId,jdbcType=BIGINT}, - - - alias = #{record.alias,jdbcType=VARCHAR}, - - - description = #{record.description,jdbcType=VARCHAR}, - - - viewer = #{record.viewer,jdbcType=VARCHAR}, - - - view_org = #{record.viewOrg,jdbcType=VARCHAR}, - - - admin = #{record.admin,jdbcType=VARCHAR}, - - - admin_org = #{record.adminOrg,jdbcType=VARCHAR}, - - - is_open = #{record.isOpen,jdbcType=INTEGER}, - - - created_by = #{record.createdBy,jdbcType=VARCHAR}, - - - created_at = #{record.createdAt,jdbcType=TIMESTAMP}, - - - updated_by = #{record.updatedBy,jdbcType=VARCHAR}, - - - updated_at = #{record.updatedAt,jdbcType=TIMESTAMP}, - - - drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, - - - status = #{record.status,jdbcType=INTEGER}, - - - entity = #{record.entity,jdbcType=LONGVARCHAR}, - - - - - - - - update s2_model - set id = #{record.id,jdbcType=BIGINT}, - name = #{record.name,jdbcType=VARCHAR}, - biz_name = #{record.bizName,jdbcType=VARCHAR}, - domain_id = #{record.domainId,jdbcType=BIGINT}, - alias = #{record.alias,jdbcType=VARCHAR}, - description = #{record.description,jdbcType=VARCHAR}, - viewer = #{record.viewer,jdbcType=VARCHAR}, - view_org = #{record.viewOrg,jdbcType=VARCHAR}, - admin = #{record.admin,jdbcType=VARCHAR}, - admin_org = #{record.adminOrg,jdbcType=VARCHAR}, - is_open = #{record.isOpen,jdbcType=INTEGER}, - created_by = #{record.createdBy,jdbcType=VARCHAR}, - created_at = #{record.createdAt,jdbcType=TIMESTAMP}, - updated_by = #{record.updatedBy,jdbcType=VARCHAR}, - updated_at = #{record.updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=INTEGER}, - entity = #{record.entity,jdbcType=LONGVARCHAR} - - - - - - update s2_model - set id = #{record.id,jdbcType=BIGINT}, - name = #{record.name,jdbcType=VARCHAR}, - biz_name = #{record.bizName,jdbcType=VARCHAR}, - domain_id = #{record.domainId,jdbcType=BIGINT}, - alias = #{record.alias,jdbcType=VARCHAR}, - description = #{record.description,jdbcType=VARCHAR}, - viewer = #{record.viewer,jdbcType=VARCHAR}, - view_org = #{record.viewOrg,jdbcType=VARCHAR}, - admin = #{record.admin,jdbcType=VARCHAR}, - admin_org = #{record.adminOrg,jdbcType=VARCHAR}, - is_open = #{record.isOpen,jdbcType=INTEGER}, - created_by = #{record.createdBy,jdbcType=VARCHAR}, - created_at = #{record.createdAt,jdbcType=TIMESTAMP}, - updated_by = #{record.updatedBy,jdbcType=VARCHAR}, - updated_at = #{record.updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=INTEGER} - - - - - - update s2_model - - - name = #{name,jdbcType=VARCHAR}, - - - biz_name = #{bizName,jdbcType=VARCHAR}, - - - domain_id = #{domainId,jdbcType=BIGINT}, - - - alias = #{alias,jdbcType=VARCHAR}, - - - description = #{description,jdbcType=VARCHAR}, - - - viewer = #{viewer,jdbcType=VARCHAR}, - - - view_org = #{viewOrg,jdbcType=VARCHAR}, - - - admin = #{admin,jdbcType=VARCHAR}, - - - admin_org = #{adminOrg,jdbcType=VARCHAR}, - - - is_open = #{isOpen,jdbcType=INTEGER}, - - - created_by = #{createdBy,jdbcType=VARCHAR}, - - - created_at = #{createdAt,jdbcType=TIMESTAMP}, - - - updated_by = #{updatedBy,jdbcType=VARCHAR}, - - - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - - - drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, - - - status = #{status,jdbcType=INTEGER}, - - - entity = #{entity,jdbcType=LONGVARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update s2_model - set name = #{name,jdbcType=VARCHAR}, - biz_name = #{bizName,jdbcType=VARCHAR}, - domain_id = #{domainId,jdbcType=BIGINT}, - alias = #{alias,jdbcType=VARCHAR}, - description = #{description,jdbcType=VARCHAR}, - viewer = #{viewer,jdbcType=VARCHAR}, - view_org = #{viewOrg,jdbcType=VARCHAR}, - admin = #{admin,jdbcType=VARCHAR}, - admin_org = #{adminOrg,jdbcType=VARCHAR}, - is_open = #{isOpen,jdbcType=INTEGER}, - created_by = #{createdBy,jdbcType=VARCHAR}, - created_at = #{createdAt,jdbcType=TIMESTAMP}, - updated_by = #{updatedBy,jdbcType=VARCHAR}, - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, - status = #{status,jdbcType=INTEGER}, - entity = #{entity,jdbcType=LONGVARCHAR} - where id = #{id,jdbcType=BIGINT} - - - update s2_model - set name = #{name,jdbcType=VARCHAR}, - biz_name = #{bizName,jdbcType=VARCHAR}, - domain_id = #{domainId,jdbcType=BIGINT}, - alias = #{alias,jdbcType=VARCHAR}, - description = #{description,jdbcType=VARCHAR}, - viewer = #{viewer,jdbcType=VARCHAR}, - view_org = #{viewOrg,jdbcType=VARCHAR}, - admin = #{admin,jdbcType=VARCHAR}, - admin_org = #{adminOrg,jdbcType=VARCHAR}, - is_open = #{isOpen,jdbcType=INTEGER}, - created_by = #{createdBy,jdbcType=VARCHAR}, - created_at = #{createdAt,jdbcType=TIMESTAMP}, - updated_by = #{updatedBy,jdbcType=VARCHAR}, - updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, - status = #{status,jdbcType=INTEGER} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index 10848caba..b118cf481 100644 --- a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -5,7 +5,6 @@ - @@ -67,8 +66,7 @@ insert into s2_dimension (name, biz_name, description, status, model_id, - type, type_params, expr, - datasource_id, created_at, created_by, + type, type_params, expr,created_at, created_by, updated_by, updated_at, semantic_type,sensitive_level, is_tag) values @@ -76,8 +74,7 @@ #{dimension.description,jdbcType=VARCHAR}, #{dimension.status,jdbcType=INTEGER}, #{dimension.modelId,jdbcType=BIGINT}, #{dimension.type,jdbcType=VARCHAR}, #{dimension.typeParams,jdbcType=VARCHAR}, - #{dimension.expr,jdbcType=VARCHAR}, - #{dimension.datasourceId,jdbcType=BIGINT}, #{dimension.createdAt,jdbcType=TIMESTAMP}, + #{dimension.expr,jdbcType=VARCHAR}, #{dimension.createdAt,jdbcType=TIMESTAMP}, #{dimension.createdBy,jdbcType=VARCHAR}, #{dimension.updatedBy,jdbcType=VARCHAR}, #{dimension.updatedAt,jdbcType=TIMESTAMP}, #{dimension.semanticType,jdbcType=VARCHAR}, @@ -161,9 +158,6 @@ and created_by = #{createdBy} - - and datasource_id = #{datasourceId} - and is_tag = 1 diff --git a/semantic/model/src/main/resources/mapper/custom/ModelDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/ModelDOCustomMapper.xml new file mode 100644 index 000000000..b79cbfda7 --- /dev/null +++ b/semantic/model/src/main/resources/mapper/custom/ModelDOCustomMapper.xml @@ -0,0 +1,18 @@ + + + + + + + update s2_model + set + status = #{model.status,jdbcType=INTEGER}, + updated_by = #{model.updatedBy,jdbcType=VARCHAR}, + updated_at = #{model.updatedAt,jdbcType=TIMESTAMP} + where id = #{model.id,jdbcType=BIGINT} + + + + + diff --git a/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImplTest.java b/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImplTest.java deleted file mode 100644 index e034cfd73..000000000 --- a/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImplTest.java +++ /dev/null @@ -1,218 +0,0 @@ -package com.tencent.supersonic.semantic.model.application; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; -import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum; -import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; -import com.tencent.supersonic.semantic.api.model.pojo.Dim; -import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.semantic.api.model.pojo.Identify; -import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.request.DatasourceReq; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.semantic.model.domain.DatabaseService; -import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.MetricService; -import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; -import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository; -import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; -import com.tencent.supersonic.semantic.model.domain.utils.DatasourceConverter; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import java.util.ArrayList; -import java.util.List; -import static org.mockito.Mockito.when; - -class DatasourceServiceImplTest { - - @Test - void createDatasource() throws Exception { - MetricService metricService = Mockito.mock(MetricService.class); - DimensionService dimensionService = Mockito.mock(DimensionService.class); - DatasourceRepository datasourceRepository = Mockito.mock(DatasourceRepository.class); - DatabaseService databaseService = Mockito.mock(DatabaseService.class); - DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); - DatasourceServiceImpl datasourceService = new DatasourceServiceImpl(datasourceRepository, databaseService, - dimensionService, metricService, dateInfoRepository); - DatasourceResp actualDatasourceResp = datasourceService.createDatasource( - mockDatasourceReq(), User.getFakeUser()); - DatasourceResp expectedDatasourceResp = buildExpectedDatasourceResp(); - Assertions.assertEquals(expectedDatasourceResp, actualDatasourceResp); - } - - @Test - void updateDatasource() throws Exception { - MetricService metricService = Mockito.mock(MetricService.class); - DimensionService dimensionService = Mockito.mock(DimensionService.class); - DatasourceRepository datasourceRepository = Mockito.mock(DatasourceRepository.class); - DatabaseService databaseService = Mockito.mock(DatabaseService.class); - DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); - DatasourceServiceImpl datasourceService = new DatasourceServiceImpl(datasourceRepository, databaseService, - dimensionService, metricService, dateInfoRepository); - DatasourceReq datasourceReq = mockDatasourceReq_update(); - DatasourceDO datasourceDO = DatasourceConverter.convert(mockDatasourceReq(), User.getFakeUser()); - when(datasourceRepository.getDatasourceById(datasourceReq.getId())).thenReturn(datasourceDO); - User user = User.getFakeUser(); - user.setName("alice"); - DatasourceResp actualDatasourceResp = datasourceService.updateDatasource(mockDatasourceReq_update(), user); - DatasourceResp expectedDatasourceResp = buildExpectedDatasourceResp_update(); - Assertions.assertEquals(expectedDatasourceResp, actualDatasourceResp); - Assertions.assertEquals("admin", actualDatasourceResp.getCreatedBy()); - Assertions.assertEquals("alice", actualDatasourceResp.getUpdatedBy()); - } - - - private DatasourceReq mockDatasourceReq() { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setName("PVUV统计"); - datasourceReq.setBizName("s2_pv_uv_statis"); - datasourceReq.setDescription("PVUV统计"); - datasourceReq.setDatabaseId(1L); - - List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); - datasourceReq.setIdentifiers(identifiers); - - List dimensions = new ArrayList<>(); - Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); - dimension1.setTypeParams(new DimensionTimeTypeParams()); - dimensions.add(dimension1); - Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0); - dimension2.setExpr("page"); - dimensions.add(dimension2); - datasourceReq.setDimensions(dimensions); - - List measures = new ArrayList<>(); - Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1); - measures.add(measure1); - - Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); - measures.add(measure2); - - datasourceReq.setMeasures(measures); - datasourceReq.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setModelId(1L); - datasourceReq.setFilterSql("where user_name = 'alice'"); - return datasourceReq; - } - - private DatasourceReq mockDatasourceReq_update() { - DatasourceReq datasourceReq = new DatasourceReq(); - datasourceReq.setId(1L); - datasourceReq.setName("PVUV统计_a"); - datasourceReq.setBizName("s2_pv_uv_statis_a"); - datasourceReq.setDescription("PVUV统计_a"); - datasourceReq.setDatabaseId(2L); - - List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名_a", IdentifyTypeEnum.primary.name(), "user_name_a")); - datasourceReq.setIdentifiers(identifiers); - - List dimensions = new ArrayList<>(); - Dim dimension1 = new Dim("", "imp_date_a", DimensionTypeEnum.time.name(), 0); - dimension1.setTypeParams(new DimensionTimeTypeParams()); - dimensions.add(dimension1); - Dim dimension2 = new Dim("", "page_a", DimensionTypeEnum.categorical.name(), 0); - dimension2.setExpr("page_a"); - dimensions.add(dimension2); - datasourceReq.setDimensions(dimensions); - - List measures = new ArrayList<>(); - Measure measure1 = new Measure("访问次数_a", "pv_a", AggOperatorEnum.SUM.name(), 1); - measures.add(measure1); - - Measure measure2 = new Measure("访问人数_a", "uv_a", AggOperatorEnum.COUNT_DISTINCT.name(), 1); - measures.add(measure2); - - datasourceReq.setMeasures(measures); - datasourceReq.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, user_name " - + "as uv_a FROM s2_pv_uv_statis"); - datasourceReq.setQueryType("sql_query"); - datasourceReq.setModelId(1L); - datasourceReq.setFilterSql("where user_name = 'tom'"); - return datasourceReq; - } - - private DatasourceResp buildExpectedDatasourceResp() { - DatasourceResp datasourceResp = new DatasourceResp(); - datasourceResp.setName("PVUV统计"); - datasourceResp.setBizName("s2_pv_uv_statis"); - datasourceResp.setDescription("PVUV统计"); - datasourceResp.setDatabaseId(1L); - DatasourceDetail datasourceDetail = new DatasourceDetail(); - List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); - datasourceDetail.setIdentifiers(identifiers); - - List dimensions = new ArrayList<>(); - Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); - dimension1.setTypeParams(new DimensionTimeTypeParams()); - dimensions.add(dimension1); - Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0); - dimension2.setExpr("page"); - dimensions.add(dimension2); - datasourceDetail.setDimensions(dimensions); - - List measures = new ArrayList<>(); - Measure measure1 = new Measure("访问次数", "s2_pv_uv_statis_pv", AggOperatorEnum.SUM.name(), 1); - measure1.setExpr("pv"); - measures.add(measure1); - - Measure measure2 = new Measure("访问人数", "s2_pv_uv_statis_uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); - measure2.setExpr("uv"); - measures.add(measure2); - - datasourceDetail.setMeasures(measures); - datasourceDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); - datasourceDetail.setQueryType("sql_query"); - datasourceResp.setDatasourceDetail(datasourceDetail); - datasourceResp.setModelId(1L); - datasourceResp.setFilterSql("where user_name = 'alice'"); - return datasourceResp; - } - - private DatasourceResp buildExpectedDatasourceResp_update() { - DatasourceResp datasourceResp = new DatasourceResp(); - datasourceResp.setName("PVUV统计_a"); - datasourceResp.setBizName("s2_pv_uv_statis_a"); - datasourceResp.setDescription("PVUV统计_a"); - datasourceResp.setDatabaseId(2L); - DatasourceDetail datasourceDetail = new DatasourceDetail(); - List identifiers = new ArrayList<>(); - identifiers.add(new Identify("用户名_a", IdentifyTypeEnum.primary.name(), "user_name_a")); - datasourceDetail.setIdentifiers(identifiers); - - List dimensions = new ArrayList<>(); - Dim dimension1 = new Dim("", "imp_date_a", DimensionTypeEnum.time.name(), 0); - dimension1.setTypeParams(new DimensionTimeTypeParams()); - dimensions.add(dimension1); - Dim dimension2 = new Dim("", "page_a", DimensionTypeEnum.categorical.name(), 0); - dimension2.setExpr("page_a"); - dimensions.add(dimension2); - datasourceDetail.setDimensions(dimensions); - - List measures = new ArrayList<>(); - Measure measure1 = new Measure("访问次数_a", "s2_pv_uv_statis_a_pv_a", - AggOperatorEnum.SUM.name(), 1); - measure1.setExpr("pv_a"); - measures.add(measure1); - - Measure measure2 = new Measure("访问人数_a", "s2_pv_uv_statis_a_uv_a", - AggOperatorEnum.COUNT_DISTINCT.name(), 1); - measure2.setExpr("uv_a"); - measures.add(measure2); - - datasourceDetail.setMeasures(measures); - datasourceDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, " - + "user_name as uv_a FROM s2_pv_uv_statis"); - datasourceDetail.setQueryType("sql_query"); - datasourceResp.setDatasourceDetail(datasourceDetail); - datasourceResp.setModelId(1L); - datasourceResp.setFilterSql("where user_name = 'tom'"); - return datasourceResp; - } - -} \ No newline at end of file diff --git a/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/ModelServiceImplTest.java b/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/ModelServiceImplTest.java new file mode 100644 index 000000000..50f9741b1 --- /dev/null +++ b/semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/ModelServiceImplTest.java @@ -0,0 +1,241 @@ +package com.tencent.supersonic.semantic.model.application; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.service.UserService; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; +import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; +import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams; +import com.tencent.supersonic.semantic.api.model.pojo.Identify; +import com.tencent.supersonic.semantic.api.model.pojo.Measure; +import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; +import com.tencent.supersonic.semantic.api.model.request.ModelReq; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.model.domain.DatabaseService; +import com.tencent.supersonic.semantic.model.domain.DimensionService; +import com.tencent.supersonic.semantic.model.domain.DomainService; +import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.ModelRelaService; +import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; +import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; +import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; +import com.tencent.supersonic.semantic.model.domain.utils.ModelConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Mockito.when; + +class ModelServiceImplTest { + + @Test + void createModel() throws Exception { + ModelRepository modelRepository = Mockito.mock(ModelRepository.class); + ModelService modelService = mockModelService(modelRepository); + ModelResp actualModelResp = modelService.createModel( + mockModelReq(), User.getFakeUser()); + ModelResp expectedModelResp = buildExpectedModelResp(); + Assertions.assertEquals(expectedModelResp, actualModelResp); + } + + @Test + void updateModel() throws Exception { + ModelRepository modelRepository = Mockito.mock(ModelRepository.class); + ModelService modelService = mockModelService(modelRepository); + ModelReq modelReq = mockModelReq_update(); + ModelDO modelDO = ModelConverter.convert(mockModelReq(), User.getFakeUser()); + when(modelRepository.getModelById(modelReq.getId())).thenReturn(modelDO); + User user = User.getFakeUser(); + user.setName("alice"); + ModelResp actualModelResp = modelService.updateModel(modelReq, user); + ModelResp expectedModelResp = buildExpectedModelResp_update(); + Assertions.assertEquals(expectedModelResp, actualModelResp); + Assertions.assertEquals("admin", actualModelResp.getCreatedBy()); + Assertions.assertEquals("alice", actualModelResp.getUpdatedBy()); + } + + + private ModelService mockModelService(ModelRepository modelRepository) { + MetricService metricService = Mockito.mock(MetricService.class); + DimensionService dimensionService = Mockito.mock(DimensionService.class); + DatabaseService databaseService = Mockito.mock(DatabaseService.class); + DomainService domainService = Mockito.mock(DomainService.class); + UserService userService = Mockito.mock(UserService.class); + ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class); + DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class); + return new ModelServiceImpl(modelRepository, databaseService, + dimensionService, metricService, modelRelaService, domainService, userService, dateInfoRepository); + } + + private ModelReq mockModelReq() { + ModelReq modelReq = new ModelReq(); + modelReq.setId(1L); + modelReq.setName("PVUV统计"); + modelReq.setBizName("s2_pv_uv_statis"); + modelReq.setDescription("PVUV统计"); + modelReq.setDatabaseId(1L); + modelReq.setAlias("访问次数统计,PVUV统计"); + modelReq.setAdmins(Lists.newArrayList("admin", "tom")); + modelReq.setViewers(Lists.newArrayList("alice", "lucy")); + ModelDetail modelDetail = new ModelDetail(); + List identifiers = new ArrayList<>(); + identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); + modelDetail.setIdentifiers(identifiers); + List dimensions = new ArrayList<>(); + Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); + dimension1.setTypeParams(new DimensionTimeTypeParams()); + dimensions.add(dimension1); + Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0); + dimension2.setExpr("page"); + dimensions.add(dimension2); + modelDetail.setDimensions(dimensions); + List measures = new ArrayList<>(); + Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1); + measures.add(measure1); + Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); + measures.add(measure2); + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setQueryType("sql_query"); + modelReq.setDomainId(1L); + modelReq.setFilterSql("where user_name = 'alice'"); + modelReq.setModelDetail(modelDetail); + return modelReq; + } + + private ModelReq mockModelReq_update() { + ModelReq modelReq = new ModelReq(); + modelReq.setId(1L); + modelReq.setName("PVUV统计_a"); + modelReq.setBizName("s2_pv_uv_statis_a"); + modelReq.setDescription("PVUV统计_a"); + modelReq.setDatabaseId(2L); + modelReq.setDomainId(1L); + modelReq.setAlias("访问次数统计,PVUV统计"); + modelReq.setAdmins(Lists.newArrayList("admin")); + modelReq.setViewers(Lists.newArrayList("alice")); + ModelDetail modelDetail = new ModelDetail(); + List identifiers = new ArrayList<>(); + identifiers.add(new Identify("用户名_a", IdentifyTypeEnum.primary.name(), "user_name_a")); + modelDetail.setIdentifiers(identifiers); + + List dimensions = new ArrayList<>(); + Dim dimension1 = new Dim("", "imp_date_a", DimensionTypeEnum.time.name(), 0); + dimension1.setTypeParams(new DimensionTimeTypeParams()); + dimensions.add(dimension1); + Dim dimension2 = new Dim("", "page_a", DimensionTypeEnum.categorical.name(), 0); + dimension2.setExpr("page_a"); + dimensions.add(dimension2); + modelDetail.setDimensions(dimensions); + + List measures = new ArrayList<>(); + Measure measure1 = new Measure("访问次数_a", "pv_a", AggOperatorEnum.SUM.name(), 1); + measures.add(measure1); + + Measure measure2 = new Measure("访问人数_a", "uv_a", AggOperatorEnum.COUNT_DISTINCT.name(), 1); + measures.add(measure2); + + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, user_name " + + "as uv_a FROM s2_pv_uv_statis"); + modelDetail.setQueryType("sql_query"); + modelReq.setDomainId(1L); + modelReq.setFilterSql("where user_name = 'tom'"); + modelReq.setModelDetail(modelDetail); + return modelReq; + } + + private ModelResp buildExpectedModelResp() { + ModelResp modelResp = new ModelResp(); + modelResp.setName("PVUV统计"); + modelResp.setBizName("s2_pv_uv_statis"); + modelResp.setDescription("PVUV统计"); + modelResp.setDatabaseId(1L); + modelResp.setDomainId(1L); + modelResp.setAlias("访问次数统计,PVUV统计"); + modelResp.setAdmins(Lists.newArrayList("admin", "tom")); + modelResp.setViewers(Lists.newArrayList("alice", "lucy")); + ModelDetail modelDetail = new ModelDetail(); + List identifiers = new ArrayList<>(); + identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name")); + modelDetail.setIdentifiers(identifiers); + + List dimensions = new ArrayList<>(); + Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0); + dimension1.setTypeParams(new DimensionTimeTypeParams()); + dimensions.add(dimension1); + Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0); + dimension2.setExpr("page"); + dimensions.add(dimension2); + modelDetail.setDimensions(dimensions); + + List measures = new ArrayList<>(); + Measure measure1 = new Measure("访问次数", "s2_pv_uv_statis_pv", AggOperatorEnum.SUM.name(), 1); + measure1.setExpr("pv"); + measures.add(measure1); + + Measure measure2 = new Measure("访问人数", "s2_pv_uv_statis_uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); + measure2.setExpr("uv"); + measures.add(measure2); + + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis"); + modelDetail.setQueryType("sql_query"); + modelResp.setModelDetail(modelDetail); + modelResp.setId(1L); + modelResp.setFilterSql("where user_name = 'alice'"); + return modelResp; + } + + private ModelResp buildExpectedModelResp_update() { + ModelResp modelResp = new ModelResp(); + modelResp.setName("PVUV统计_a"); + modelResp.setBizName("s2_pv_uv_statis_a"); + modelResp.setDescription("PVUV统计_a"); + modelResp.setDatabaseId(2L); + modelResp.setDomainId(1L); + modelResp.setAlias("访问次数统计,PVUV统计"); + modelResp.setAdmins(Lists.newArrayList("admin")); + modelResp.setViewers(Lists.newArrayList("alice")); + ModelDetail modelDetail = new ModelDetail(); + List identifiers = new ArrayList<>(); + identifiers.add(new Identify("用户名_a", IdentifyTypeEnum.primary.name(), "user_name_a")); + modelDetail.setIdentifiers(identifiers); + + List dimensions = new ArrayList<>(); + Dim dimension1 = new Dim("", "imp_date_a", DimensionTypeEnum.time.name(), 0); + dimension1.setTypeParams(new DimensionTimeTypeParams()); + dimensions.add(dimension1); + Dim dimension2 = new Dim("", "page_a", DimensionTypeEnum.categorical.name(), 0); + dimension2.setExpr("page_a"); + dimensions.add(dimension2); + modelDetail.setDimensions(dimensions); + + List measures = new ArrayList<>(); + Measure measure1 = new Measure("访问次数_a", "s2_pv_uv_statis_a_pv_a", + AggOperatorEnum.SUM.name(), 1); + measure1.setExpr("pv_a"); + measures.add(measure1); + + Measure measure2 = new Measure("访问人数_a", "s2_pv_uv_statis_a_uv_a", + AggOperatorEnum.COUNT_DISTINCT.name(), 1); + measure2.setExpr("uv_a"); + measures.add(measure2); + + modelDetail.setMeasures(measures); + modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, " + + "user_name as uv_a FROM s2_pv_uv_statis"); + modelDetail.setQueryType("sql_query"); + modelResp.setModelDetail(modelDetail); + modelResp.setId(1L); + modelResp.setFilterSql("where user_name = 'tom'"); + return modelResp; + } + +} \ No newline at end of file diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java index 9896ca73b..745896baf 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java @@ -85,8 +85,9 @@ public class MaterializationQuery implements QueryOptimizer { return; } try { - if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStatement) || Objects.isNull( - queryStructCmd.getModelId()) || Objects.isNull( + if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStatement) + || CollectionUtils.isEmpty(queryStructCmd.getModelIds()) + || Objects.isNull( queryStructCmd.getDateInfo())) { return; } @@ -172,10 +173,11 @@ public class MaterializationQuery implements QueryOptimizer { ImmutablePair timeRange = queryStructUtils.getBeginEndTime(queryStructReq); String start = timeRange.left; String end = timeRange.right; - Long modelId = queryStructReq.getModelId(); + //todo + Long modelId = 1L; List materializationResps = materializationConfService.getMaterializationByModel(modelId); - List dimensionResps = catalog.getDimensions(modelId); - List metrics = catalog.getMetrics(modelId); + List dimensionResps = catalog.getDimensions(queryStructReq.getModelIds()); + List metrics = catalog.getMetrics(queryStructReq.getModelIds()); Set fields = new HashSet<>(); if (Objects.nonNull(metricReq.getWhere()) && !metricReq.getWhere().isEmpty()) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java index 3b8ecfa4d..d597aa2aa 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java @@ -41,6 +41,7 @@ public class CalciteSqlParser implements SqlParser { semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDimension(semanticModel.getDimensionMap()); semanticSchema.setMetric(semanticModel.getMetrics()); + semanticSchema.setJoinRelations(semanticModel.getJoinRelations()); return semanticSchema; } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java index ef6195471..0a08ad4ab 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java @@ -3,7 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; +import com.tencent.supersonic.common.pojo.ModelRela; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl; @@ -17,12 +19,14 @@ import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DimensionTimeTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify; +import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MetricTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -34,6 +38,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.EnableCaching; @@ -57,20 +62,27 @@ public class SemanticSchemaManager { public SemanticModel reload(String rootPath) { SemanticModel semanticModel = new SemanticModel(); semanticModel.setRootPath(rootPath); - Map modelFullPathMap = catalog.getModelFullPath(); - log.info("modelFullPathMap {}", modelFullPathMap); - Set modelIds = modelFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath)) - .map(Entry::getKey).collect(Collectors.toSet()); + //Map modelFullPathMap = catalog.getModelFullPath(); + //log.info("modelFullPathMap {}", modelFullPathMap); + //Set modelIds = modelFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath)) + // .map(Entry::getKey).collect(Collectors.toSet()); + Set modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim())) + .collect(Collectors.toSet()); if (modelIds.isEmpty()) { log.error("get modelIds empty {}", rootPath); return semanticModel; } Map> dimensionYamlTpls = new HashMap<>(); - List datasourceYamlTpls = new ArrayList<>(); + List dataModelYamlTpls = new ArrayList<>(); List metricYamlTpls = new ArrayList<>(); - catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls); - if (!datasourceYamlTpls.isEmpty()) { - Map dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d)) + Map modelIdName = new HashMap<>(); + catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); + List modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); + if (!CollectionUtils.isEmpty(modelRelas)) { + semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); + } + if (!dataModelYamlTpls.isEmpty()) { + Map dataSourceMap = dataModelYamlTpls.stream().map(d -> getDatasource(d)) .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); semanticModel.setDatasourceMap(dataSourceMap); } @@ -107,7 +119,7 @@ public class SemanticSchemaManager { } - public static DataSource getDatasource(final DatasourceYamlTpl d) { + public static DataSource getDatasource(final DataModelYamlTpl d) { DataSource datasource = DataSource.builder().sourceId(d.getSourceId()).sqlQuery(d.getSqlQuery()) .name(d.getName()).tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) .measures(getMeasures(d.getMeasures())).dimensions(getDimensions(d.getDimensions())).build(); @@ -200,6 +212,25 @@ public class SemanticSchemaManager { return identifies; } + private static List getJoinRelation(List modelRelas, Map modelIdName) { + List joinRelations = new ArrayList<>(); + modelRelas.stream().forEach(r -> { + if (modelIdName.containsKey(r.getFromModelId()) && modelIdName.containsKey(r.getToModelId())) { + JoinRelation joinRelation = JoinRelation.builder().left(modelIdName.get(r.getFromModelId())) + .right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build(); + List> conditions = new ArrayList<>(); + r.getJoinConditions().stream().forEach(rr -> { + if (FilterOperatorEnum.isValueCompare(rr.getOperator())) { + conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField())); + } + }); + joinRelation.setJoinCondition(conditions); + joinRelations.add(joinRelation); + } + }); + return joinRelations; + } + public static void update(SemanticSchema schema, List metric) throws Exception { if (schema != null) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java index 1d0c328be..797a9f537 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java @@ -15,18 +15,19 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRen import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender; import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.SourceRender; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; + import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Objects; import java.util.Stack; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; public class AggPlanner implements Planner { - private MetricReq metricCommand; + private MetricReq metricReq; private SemanticSchema schema; private SqlValidatorScope scope; private Stack dataSets = new Stack<>(); @@ -61,13 +62,13 @@ public class AggPlanner implements Planner { while (it.hasNext()) { Renderer renderer = it.next(); if (previous != null) { - previous.render(metricCommand, datasource, scope, schema, !isAgg); + previous.render(metricReq, datasource, scope, schema, !isAgg); renderer.setTable(previous.builderAs(DataSourceNode.getNames(datasource) + "_" + String.valueOf(i))); i++; } previous = renderer; } - builders.getLast().render(metricCommand, datasource, scope, schema, !isAgg); + builders.getLast().render(metricReq, datasource, scope, schema, !isAgg); parserNode = builders.getLast().builder(); @@ -75,7 +76,7 @@ public class AggPlanner implements Planner { private List getMatchDataSource(SqlValidatorScope scope) throws Exception { - return DataSourceNode.getMatchDataSources(scope, schema, metricCommand); + return DataSourceNode.getMatchDataSources(scope, schema, metricReq); } private boolean getAgg(DataSource dataSource) { @@ -85,7 +86,7 @@ public class AggPlanner implements Planner { // default by dataSource time aggregation if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime().equalsIgnoreCase( Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { - if (!metricCommand.isNativeQuery()) { + if (!metricReq.isNativeQuery()) { return true; } } @@ -95,15 +96,15 @@ public class AggPlanner implements Planner { @Override public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception { - this.metricCommand = queryStatement.getMetricReq(); - if (metricCommand.getMetrics() == null) { - metricCommand.setMetrics(new ArrayList<>()); + this.metricReq = queryStatement.getMetricReq(); + if (metricReq.getMetrics() == null) { + metricReq.setMetrics(new ArrayList<>()); } - if (metricCommand.getDimensions() == null) { - metricCommand.setDimensions(new ArrayList<>()); + if (metricReq.getDimensions() == null) { + metricReq.setDimensions(new ArrayList<>()); } - if (metricCommand.getLimit() == null) { - metricCommand.setLimit(0L); + if (metricReq.getLimit() == null) { + metricReq.setLimit(0L); } this.aggOption = aggOption; // build a parse Node diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/JoinRelation.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/JoinRelation.java new file mode 100644 index 000000000..a374a2747 --- /dev/null +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/JoinRelation.java @@ -0,0 +1,17 @@ +package com.tencent.supersonic.semantic.query.parser.calcite.s2sql; + +import java.util.List; +import lombok.Builder; +import lombok.Data; +import org.apache.commons.lang3.tuple.Triple; + +@Data +@Builder +public class JoinRelation { + + private String left; + private String right; + private String joinType; + private List> joinCondition; + +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java index 25909aac6..43c3c0b79 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java @@ -14,4 +14,5 @@ public class SemanticModel { private Map datasourceMap = new HashMap<>(); private Map> dimensionMap = new HashMap<>(); private List materializationList = new ArrayList<>(); + private List joinRelations; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java index d98c3a090..ce81c167f 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java @@ -3,17 +3,19 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaVersion; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class SemanticSchema extends AbstractSchema { private final String rootPath; @@ -21,6 +23,8 @@ public class SemanticSchema extends AbstractSchema { private SemanticModel semanticModel = new SemanticModel(); + private List joinRelations; + private SemanticSchema(String rootPath, Map tableMap) { this.rootPath = rootPath; @@ -84,6 +88,14 @@ public class SemanticSchema extends AbstractSchema { return semanticModel.getMaterializationList(); } + public void setJoinRelations(List joinRelations) { + this.joinRelations = joinRelations; + } + + public List getJoinRelations() { + return joinRelations; + } + public static final class Builder { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java index b07fb9a97..7a0fc272a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java @@ -1,13 +1,26 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; +import com.google.common.collect.Lists; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension; +import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.extend.LateralViewExplodeNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -19,16 +32,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; @Slf4j public class DataSourceNode extends SemanticNode { @@ -159,21 +162,26 @@ public class DataSourceNode extends SemanticNode { return dataSources; } // find all dataSource has the same identifiers - Set baseIdentifiers = baseDataSource.getIdentifiers().stream().map(i -> i.getName()) - .collect(Collectors.toSet()); - if (baseIdentifiers.isEmpty()) { - throw new Exception("datasource error : " + baseDataSource.getName() + " miss identifier"); - } - List linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures, + List linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension, measures, baseDataSource, schema); - if (linkDataSources.isEmpty()) { - throw new Exception( - String.format("not find the match datasource : dimension[%s],measure[%s]", queryDimension, - measures)); + if (CollectionUtils.isEmpty(linkDataSources)) { + log.info("baseDataSource get by identifiers "); + Set baseIdentifiers = baseDataSource.getIdentifiers().stream().map(i -> i.getName()) + .collect(Collectors.toSet()); + if (baseIdentifiers.isEmpty()) { + throw new Exception("datasource error : " + baseDataSource.getName() + " miss identifier"); + } + linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures, + baseDataSource, schema); + if (linkDataSources.isEmpty()) { + throw new Exception( + String.format("not find the match datasource : dimension[%s],measure[%s]", queryDimension, + measures)); + } } log.debug("linkDataSources {}", linkDataSources); - - dataSources.addAll(linkDataSources); + return linkDataSources; + //dataSources.addAll(linkDataSources); } return dataSources; @@ -208,6 +216,69 @@ public class DataSourceNode extends SemanticNode { return isAllMatch; } + private static List getLinkDataSourcesByJoinRelation(Set queryDimension, List measures, + DataSource baseDataSource, SemanticSchema schema) { + Set linkDataSourceName = new HashSet<>(); + List linkDataSources = new ArrayList<>(); + Set before = new HashSet<>(); + before.add(baseDataSource.getName()); + if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { + for (JoinRelation joinRelation : schema.getJoinRelations()) { + if (!before.contains(joinRelation.getLeft()) && !before.contains(joinRelation.getRight())) { + continue; + } + boolean isMatch = false; + boolean isRight = before.contains(joinRelation.getLeft()); + DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight()) + : schema.getDatasource().get(joinRelation.getLeft()); + if (!queryDimension.isEmpty()) { + Set linkDimension = other.getDimensions().stream().map(dd -> dd.getName()) + .collect(Collectors.toSet()); + other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName())); + linkDimension.retainAll(queryDimension); + if (!linkDimension.isEmpty()) { + isMatch = true; + } + } + Set linkMeasure = other.getMeasures().stream().map(mm -> mm.getName()) + .collect(Collectors.toSet()); + linkMeasure.retainAll(measures); + if (!linkMeasure.isEmpty()) { + isMatch = true; + } + if (!isMatch && schema.getDimension().containsKey(other.getName())) { + Set linkDimension = schema.getDimension().get(other.getName()).stream() + .map(dd -> dd.getName()) + .collect(Collectors.toSet()); + linkDimension.retainAll(queryDimension); + if (!linkDimension.isEmpty()) { + isMatch = true; + } + } + if (isMatch) { + linkDataSourceName.add(other.getName()); + before.add(other.getName()); + } + } + } + if (!CollectionUtils.isEmpty(linkDataSourceName)) { + Map orders = new HashMap<>(); + linkDataSourceName.add(baseDataSource.getName()); + orders.put(baseDataSource.getName(), 0L); + for (JoinRelation joinRelation : schema.getJoinRelations()) { + if (linkDataSourceName.contains(joinRelation.getLeft()) && linkDataSourceName.contains( + joinRelation.getRight())) { + orders.put(joinRelation.getLeft(), 0L); + orders.put(joinRelation.getRight(), 1L); + } + } + orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> { + linkDataSources.add(schema.getDatasource().get(d.getKey())); + }); + } + return linkDataSources; + } + private static List getLinkDataSources(Set baseIdentifiers, Set queryDimension, List measures, @@ -258,6 +329,12 @@ public class DataSourceNode extends SemanticNode { for (String linkName : linkDataSourceName) { linkDataSources.add(schema.getDatasource().get(linkName)); } - return linkDataSources; + if (!CollectionUtils.isEmpty(linkDataSources)) { + List all = new ArrayList<>(); + all.add(baseDataSource); + all.addAll(linkDataSources); + return all; + } + return Lists.newArrayList(); } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java index b3f41bf98..015a5a56c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java @@ -2,24 +2,21 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWriterConfig; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; @@ -29,6 +26,15 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + public abstract class SemanticNode { public static SqlNode parse(String expression, SqlValidatorScope scope) throws Exception { @@ -130,6 +136,46 @@ public abstract class SemanticNode { sqlValidator.validate(SqlParser.create(sql, SqlParser.Config.DEFAULT).parseStmt()), false, true).rel; } + public static SqlBinaryOperator getBinaryOperator(String val) { + if (val.equals("=")) { + return SqlStdOperatorTable.EQUALS; + } + if (val.equals(">")) { + return SqlStdOperatorTable.GREATER_THAN; + } + if (val.equals(">=")) { + return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL; + } + if (val.equals("<")) { + return SqlStdOperatorTable.LESS_THAN; + } + if (val.equals("<=")) { + return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; + } + if (val.equals("!=")) { + return SqlStdOperatorTable.NOT_EQUALS; + } + return SqlStdOperatorTable.EQUALS; + } + + public static SqlLiteral getJoinSqlLiteral(String joinType) { + if (Objects.nonNull(joinType) && !joinType.isEmpty()) { + if (joinType.toLowerCase().contains(JoinType.INNER.lowerName)) { + return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO); + } + if (joinType.toLowerCase().contains(JoinType.LEFT.lowerName)) { + return SqlLiteral.createSymbol(JoinType.LEFT, SqlParserPos.ZERO); + } + if (joinType.toLowerCase().contains(JoinType.RIGHT.lowerName)) { + return SqlLiteral.createSymbol(JoinType.RIGHT, SqlParserPos.ZERO); + } + if (joinType.toLowerCase().contains(JoinType.FULL.lowerName)) { + return SqlLiteral.createSymbol(JoinType.FULL, SqlParserPos.ZERO); + } + } + return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO); + } + public void accept(Optimization optimization) { optimization.visit(this); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java index e388ab557..522609265 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java @@ -5,10 +5,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify; - import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type; +import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType; - import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; @@ -19,6 +18,18 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.JoinConditionType; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.util.CollectionUtils; + import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; @@ -28,21 +39,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.JoinConditionType; -import org.apache.calcite.sql.JoinType; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlJoin; -import org.apache.calcite.sql.SqlLiteral; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; @Slf4j public class JoinRender extends Renderer { @@ -51,7 +52,7 @@ public class JoinRender extends Renderer { public void render(MetricReq metricCommand, List dataSources, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); - dataSources = getOrderSource(dataSources); + //dataSources = getOrderSource(dataSources); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); if (queryWhere != null && !queryWhere.isEmpty()) { @@ -68,6 +69,7 @@ public class JoinRender extends Renderer { TableView filterView = new TableView(); Map innerSelect = new HashMap<>(); Set filterDimension = new HashSet<>(); + Map beforeSources = new HashMap<>(); for (int i = 0; i < dataSources.size(); i++) { final DataSource dataSource = dataSources.get(i); @@ -112,18 +114,12 @@ public class JoinRender extends Renderer { if (left == null) { leftTable = tableView; left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)); + beforeSources.put(dataSource.getName(), leftTable.getAlias()); continue; } - - left = new SqlJoin( - SqlParserPos.ZERO, - left, - SqlLiteral.createBoolean(false, SqlParserPos.ZERO), - SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO), - SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)), - SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), - getCondition(leftTable, tableView, dataSource, schema, scope)); + left = buildJoin(left, leftTable, tableView, beforeSources, dataSource, schema, scope); leftTable = tableView; + beforeSources.put(dataSource.getName(), tableView.getAlias()); } for (Map.Entry entry : innerSelect.entrySet()) { @@ -258,6 +254,76 @@ public class JoinRender extends Renderer { return SemanticNode.getTable(tableView.getTable()); } + private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map before, + DataSource dataSource, + SemanticSchema schema, SqlValidatorScope scope) + throws Exception { + SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope); + SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); + if (!TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) && !TimePartType.ZIPPER.equals( + tableView.getDataSource().getTimePartType())) { + JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema); + if (!CollectionUtils.isEmpty(matchJoinRelation.getJoinCondition())) { + sqlLiteral = SemanticNode.getJoinSqlLiteral(matchJoinRelation.getJoinType()); + condition = getCondition(matchJoinRelation, scope); + } + } + return new SqlJoin( + SqlParserPos.ZERO, + left, + SqlLiteral.createBoolean(false, SqlParserPos.ZERO), + sqlLiteral, + SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)), + SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), + condition + ); + } + + private JoinRelation getMatchJoinRelation(Map before, TableView tableView, SemanticSchema schema) { + JoinRelation matchJoinRelation = JoinRelation.builder().build(); + if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { + for (JoinRelation joinRelation : schema.getJoinRelations()) { + if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName()) + && before.containsKey(joinRelation.getLeft())) { + matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() + .map(r -> Triple.of(before.get(joinRelation.getLeft()) + "." + r.getLeft(), + r.getMiddle(), tableView.getAlias() + "." + r.getRight())).collect( + Collectors.toList())); + matchJoinRelation.setJoinType(joinRelation.getJoinType()); + } + } + + } + return matchJoinRelation; + } + + + private SqlNode getCondition(JoinRelation joinRelation, + SqlValidatorScope scope) throws Exception { + SqlNode condition = null; + for (Triple con : joinRelation.getJoinCondition()) { + List ons = new ArrayList<>(); + ons.add(SemanticNode.parse(con.getLeft(), scope)); + ons.add(SemanticNode.parse(con.getRight(), scope)); + if (Objects.isNull(condition)) { + condition = new SqlBasicCall( + SemanticNode.getBinaryOperator(con.getMiddle()), + ons, + SqlParserPos.ZERO, null); + continue; + } + SqlNode addCondition = new SqlBasicCall( + SemanticNode.getBinaryOperator(con.getMiddle()), + ons, + SqlParserPos.ZERO, null); + condition = new SqlBasicCall( + SqlStdOperatorTable.AND, + new ArrayList<>(Arrays.asList(condition, addCondition)), + SqlParserPos.ZERO, null); + } + return condition; + } + private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) throws Exception { if (TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) || TimePartType.ZIPPER.equals( @@ -318,8 +384,7 @@ public class JoinRender extends Renderer { }); int cnt = dataSources.size(); List>> dataSourceIdentifyList = dataSourceIdentifies.entrySet().stream() - .collect( - Collectors.toList()); + .collect(Collectors.toList()); for (int i = 0; i < cnt; i++) { for (int j = i + 1; j < cnt; j++) { Set primaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(), @@ -384,6 +449,7 @@ public class JoinRender extends Renderer { orders.poll(); visited.put(id, false); } + private void addZipperField(DataSource dataSource, List fields) { if (TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { dataSource.getDimensions().stream() diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java index 4ce00e4a1..39be0bc86 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; @@ -15,20 +16,20 @@ import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.query.parser.SemanticConverter; import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; -import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; import com.tencent.supersonic.semantic.query.utils.SqlGenerateUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + @Component("CalculateAggConverter") @Slf4j @@ -66,7 +67,7 @@ public class CalculateAggConverter implements SemanticConverter { return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version); } ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId())); + sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds())); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); @@ -120,7 +121,7 @@ public class CalculateAggConverter implements SemanticConverter { @Override public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception { - DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelId()); + DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelIds().get(0)); ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd, EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); sqlCommend.setSql(parseSqlReq.getSql()); @@ -150,7 +151,7 @@ public class CalculateAggConverter implements SemanticConverter { throws Exception { check(queryStructCmd); ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId())); + sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds())); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java index f16efd084..2b0b733e6 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java @@ -26,7 +26,7 @@ public class DefaultDimValueConverter implements SemanticConverter { @Override public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception { - List dimensionResps = catalog.getDimensions(queryStructCmd.getModelId()); + List dimensionResps = catalog.getDimensions(queryStructCmd.getModelIds()); //dimension which has default values dimensionResps = dimensionResps.stream() .filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues())) diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java index 9853427c0..23ed60604 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java @@ -33,9 +33,8 @@ public class MetricCheckConverter implements SemanticConverter { @Override public void converter(Catalog catalog, QueryStructReq queryStructReq, ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception { - Long modelId = queryStructReq.getModelId(); - List metricResps = catalog.getMetrics(modelId); - List dimensionResps = catalog.getDimensions(modelId); + List metricResps = catalog.getMetrics(queryStructReq.getModelIds()); + List dimensionResps = catalog.getDimensions(queryStructReq.getModelIds()); Map dimensionMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getId, d -> d)); List metricBizNames = queryStructReq.getMetrics(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java deleted file mode 100644 index c919058da..000000000 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.tencent.supersonic.semantic.query.parser.convert; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.semantic.api.model.pojo.Identify; -import com.tencent.supersonic.semantic.api.model.pojo.Measure; -import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; -import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; -import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; -import com.tencent.supersonic.semantic.model.domain.Catalog; -import com.tencent.supersonic.semantic.query.parser.SemanticConverter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -@Slf4j -@Component("MultiSourceJoin") -public class MultiSourceJoin implements SemanticConverter { - - private final Catalog catalog; - - - public MultiSourceJoin(Catalog catalog) { - this.catalog = catalog; - } - - - public void buildJoinPrefix(QueryStructReq queryStructCmd) { - List groups = queryStructCmd.getGroups(); - List aggregators = queryStructCmd.getAggregators(); - List filters = queryStructCmd.getOriginalFilter(); - List fields = Lists.newArrayList(); - fields.addAll(groups); - fields.addAll(filters.stream().map(Filter::getBizName).collect(Collectors.toList())); - - if (CollectionUtils.isEmpty(groups) || CollectionUtils.isEmpty(aggregators)) { - return; - } - Long modelId = queryStructCmd.getModelId(); - List aggs = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList()); - Map dimensionMap = catalog.getDimensions(modelId).stream() - .filter(dimensionDesc -> fields.contains(dimensionDesc.getBizName())) - .collect(Collectors.toMap(DimensionResp::getBizName, dimensionDesc -> dimensionDesc)); - List metricDescList = catalog.getMetrics(modelId).stream() - .filter(metricDesc -> aggs.contains(metricDesc.getBizName())) - .collect(Collectors.toList()); - Map datasourceMap = catalog.getDatasourceList(modelId) - .stream().collect(Collectors.toMap(DatasourceResp::getId, datasource -> datasource)); - //check groups filters and aggs is in same datasource - if (!isInSameDatasource(new ArrayList<>(dimensionMap.values()), metricDescList)) { - List groupsWithPrefix = Lists.newArrayList(); - for (String group : groups) { - DimensionResp dimensionDesc = dimensionMap.get(group); - if (dimensionDesc == null) { - groupsWithPrefix.add(group); - continue; - } - String joinKeyName = getJoinKey(datasourceMap, dimensionDesc.getDatasourceId()); - if (joinKeyName.equalsIgnoreCase(group)) { - groupsWithPrefix.add(group); - } else { - String groupWithPrefix = String.format("%s__%s", joinKeyName, group); - groupsWithPrefix.add(groupWithPrefix); - } - } - List filtersWithPrefix = Lists.newArrayList(); - for (Filter filter : filters) { - DimensionResp dimensionDesc = dimensionMap.get(filter.getBizName()); - if (dimensionDesc == null) { - filtersWithPrefix.add(filter); - continue; - } - String joinKeyName = getJoinKey(datasourceMap, dimensionDesc.getDatasourceId()); - if (joinKeyName.equalsIgnoreCase(filter.getBizName())) { - filtersWithPrefix.add(filter); - } else { - String filterWithPrefix = String.format("%s__%s", joinKeyName, filter.getBizName()); - filter.setBizName(filterWithPrefix); - filtersWithPrefix.add(filter); - } - } - queryStructCmd.setGroups(groupsWithPrefix); - queryStructCmd.setDimensionFilters(filtersWithPrefix); - } - } - - - private String getJoinKey(Map datasourceMap, Long datasourceId) { - DatasourceResp datasourceDesc = datasourceMap.get(datasourceId); - List identifies = datasourceDesc.getDatasourceDetail().getIdentifiers(); - - Optional identifyOptional = identifies.stream() - .filter(identify -> identify.getType().equalsIgnoreCase("primary")).findFirst(); - if (identifyOptional.isPresent()) { - return identifyOptional.get().getBizName(); - } - return ""; - } - - - private boolean isInSameDatasource(List dimensionDescs, List metricDescs) { - Set datasourceIdSet = Sets.newHashSet(); - datasourceIdSet.addAll(dimensionDescs.stream().map(DimensionResp::getDatasourceId).filter(Objects::nonNull) - .collect(Collectors.toSet())); - datasourceIdSet.addAll( - metricDescs.stream().flatMap(metricDesc -> metricDesc.getTypeParams().getMeasures().stream()) - .map(Measure::getDatasourceId).filter(Objects::nonNull).collect(Collectors.toList())); - log.info("[multi source join] datasource id:{}", datasourceIdSet); - return datasourceIdSet.size() <= 1; - - } - - @Override - public boolean accept(QueryStructReq queryStructCmd) { - return true; - } - - @Override - public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, - MetricReq metricCommand) throws Exception { - buildJoinPrefix(queryStructCmd); - } -} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java index bbc77ed6a..b99278d42 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java @@ -19,7 +19,6 @@ import org.springframework.util.CollectionUtils; public class ParserDefaultConverter implements SemanticConverter { - private final CalculateAggConverter calculateCoverterAgg; private final QueryStructUtils queryStructUtils; @@ -57,20 +56,21 @@ public class ParserDefaultConverter implements SemanticConverter { sqlCommend.setVariables(queryStructCmd.getParams().stream() .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); sqlCommend.setLimit(queryStructCmd.getLimit()); - String rootPath = catalog.getModelFullPath(queryStructCmd.getModelId()); + String rootPath = catalog.getModelFullPath(queryStructCmd.getModelIds()); sqlCommend.setRootPath(rootPath); // todo tmp delete // support detail query if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) { - String internalMetricName = queryStructUtils.generateInternalMetricName( - queryStructCmd.getModelId(), queryStructCmd.getGroups()); - sqlCommend.getMetrics().add(internalMetricName); + for (Long modelId : queryStructCmd.getModelIds()) { + String internalMetricName = queryStructUtils.generateInternalMetricName( + modelId, queryStructCmd.getGroups()); + sqlCommend.getMetrics().add(internalMetricName); + } } return sqlCommend; } - } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java index 20c1c747d..fd2c45ecd 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java @@ -26,15 +26,6 @@ import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -43,6 +34,15 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + @Component @Slf4j public class QueryReqConverter { @@ -57,13 +57,13 @@ public class QueryReqConverter { @Autowired private Catalog catalog; - public QueryStatement convert(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception { + public QueryStatement convert(QueryS2SQLReq databaseReq, List modelSchemaResps) throws Exception { - if (Objects.isNull(modelSchemaResp)) { + if (CollectionUtils.isEmpty(modelSchemaResps)) { return new QueryStatement(); } //1.convert name to bizName - convertNameToBizName(databaseReq, modelSchemaResp); + convertNameToBizName(databaseReq, modelSchemaResps); //2.functionName corrector functionNameCorrector(databaseReq); //3.correct tableName @@ -75,21 +75,23 @@ public class QueryReqConverter { } //4.build MetricTables List allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql()); - List metrics = getMetrics(modelSchemaResp, allFields); + List metrics = getMetrics(modelSchemaResps, allFields); QueryStructReq queryStructCmd = new QueryStructReq(); MetricTable metricTable = new MetricTable(); metricTable.setMetrics(metrics); - Set dimensions = getDimensions(modelSchemaResp, allFields); + Set dimensions = getDimensions(modelSchemaResps, allFields); metricTable.setDimensions(new ArrayList<>(dimensions)); metricTable.setAlias(tableName.toLowerCase()); // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { - metricTable.setMetrics(new ArrayList<>(Arrays.asList( - queryStructUtils.generateInternalMetricName(databaseReq.getModelId(), - metricTable.getDimensions())))); + metricTable.setMetrics(new ArrayList<>()); + for (Long modelId : databaseReq.getModelIds()) { + metricTable.getMetrics().add(queryStructUtils.generateInternalMetricName(modelId, + metricTable.getDimensions())); + } } else { queryStructCmd.setAggregators( metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( @@ -102,9 +104,10 @@ public class QueryReqConverter { //4.build ParseSqlReq ParseSqlReq result = new ParseSqlReq(); BeanUtils.copyProperties(databaseReq, result); - result.setRootPath(domainService.getModelFullPathMap().get(databaseReq.getModelId())); + + result.setRootPath(catalog.getModelFullPath(databaseReq.getModelIds())); result.setTables(tables); - DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); + DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0)); if (!queryStructUtils.isSupportWith(EngineTypeEnum.valueOf(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); @@ -112,7 +115,7 @@ public class QueryReqConverter { } //5.physicalSql by ParseSqlReq queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql())); - queryStructCmd.setModelId(databaseReq.getModelId()); + queryStructCmd.setModelIds(databaseReq.getModelIds().stream().collect(Collectors.toSet())); queryStructCmd.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd); QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result); @@ -132,8 +135,8 @@ public class QueryReqConverter { return AggOption.DEFAULT; } - private void convertNameToBizName(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) { - Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp); + private void convertNameToBizName(QueryS2SQLReq databaseReq, List modelSchemaResps) { + Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); @@ -141,26 +144,28 @@ public class QueryReqConverter { databaseReq.setSql(replaceFields); } - private Set getDimensions(ModelSchemaResp modelSchemaResp, List allFields) { - Map dimensionLowerToNameMap = modelSchemaResp.getDimensions().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName())); + private Set getDimensions(List modelSchemaResps, List allFields) { + Map dimensionLowerToNameMap = modelSchemaResps.stream().flatMap(modelSchemaResp + -> modelSchemaResp.getDimensions().stream()) + .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName)); Map internalLowerToNameMap = QueryStructUtils.internalCols.stream() - .collect(Collectors.toMap(a -> a.toLowerCase(), a -> a)); + .collect(Collectors.toMap(String::toLowerCase, a -> a)); dimensionLowerToNameMap.putAll(internalLowerToNameMap); return allFields.stream() .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet()); } - private List getMetrics(ModelSchemaResp modelSchemaResp, List allFields) { - Map metricLowerToNameMap = modelSchemaResp.getMetrics().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName())); + private List getMetrics(List modelSchemaResps, List allFields) { + Map metricLowerToNameMap = modelSchemaResps.stream() + .flatMap(modelSchemaResp -> modelSchemaResp.getMetrics().stream()) + .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName)); return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); } private void functionNameCorrector(QueryS2SQLReq databaseReq) { - DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); + DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0)); if (Objects.isNull(database) || Objects.isNull(database.getType())) { return; } @@ -174,16 +179,17 @@ public class QueryReqConverter { } } - protected Map getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) { + protected Map getFieldNameToBizNameMap(List modelSchemaResps) { // support fieldName and field alias to bizName - Map dimensionResults = modelSchemaResp.getDimensions().stream() + Map dimensionResults = modelSchemaResps.stream().flatMap(modelSchemaResp + -> modelSchemaResp.getDimensions().stream()) .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1)); + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - Map metricResults = modelSchemaResp.getMetrics().stream() + Map metricResults = modelSchemaResps.stream().flatMap(modelSchemaResp + -> modelSchemaResp.getMetrics().stream()) .flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1)); - + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); @@ -204,8 +210,10 @@ public class QueryReqConverter { } public void correctTableName(QueryS2SQLReq databaseReq) { - String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), - Constants.TABLE_PREFIX + databaseReq.getModelId()); + String sql = databaseReq.getSql(); + for (Long modelId : databaseReq.getModelIds()) { + sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId); + } databaseReq.setSql(sql); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java index 18787919e..95c24d982 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java @@ -2,14 +2,15 @@ package com.tencent.supersonic.semantic.query.persistence.pojo; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; -import java.util.List; import lombok.Data; import org.apache.commons.lang3.tuple.ImmutablePair; +import java.util.List; + @Data public class QueryStatement { - private Long modelId = 0L; + private List modelIds; private String sql = ""; private String sourceId = ""; private String errMsg = ""; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java index dc9f83860..398b3b331 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java @@ -9,23 +9,24 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; -import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; -import com.tencent.supersonic.semantic.api.model.response.DomainResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.query.service.MaterializationService; import com.tencent.supersonic.semantic.query.service.SchemaService; -import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; +import java.util.List; @RestController @RequestMapping("/api/semantic/schema") diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java index 86eb07d82..cda58a41a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; @@ -16,7 +16,6 @@ import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; @@ -26,20 +25,18 @@ import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import lombok.extern.slf4j.Slf4j; -import org.assertj.core.util.Sets; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import java.text.SimpleDateFormat; -import java.util.List; import java.util.ArrayList; -import java.util.Map; import java.util.HashMap; -import java.util.Set; import java.util.HashSet; - +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -57,64 +54,62 @@ public class AuthCommonService { @Autowired private ModelService modelService; - public boolean doModelAdmin(User user, Long modelId) { + public boolean doModelAdmin(User user, List modelIds) { List modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); if (CollectionUtils.isEmpty(modelListAdmin)) { return false; } else { - Map> id2modelResp = modelListAdmin.stream() - .collect(Collectors.groupingBy(SchemaItem::getId)); - return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId); + Set modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet()); + return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds); } } - public void doModelVisible(User user, Long modelId) { + public void doModelVisible(User user, List modelIds) { Boolean visible = true; List modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE); if (CollectionUtils.isEmpty(modelListVisible)) { visible = false; } else { - Map> id2domainDesc = modelListVisible.stream() - .collect(Collectors.groupingBy(SchemaItem::getId)); - if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) { + Set modelVisibles = modelListVisible.stream().map(ModelResp::getId).collect(Collectors.toSet()); + if (!CollectionUtils.isEmpty(modelVisibles) && !modelVisibles.containsAll(modelIds)) { visible = false; } } if (!visible) { - ModelResp modelResp = modelService.getModel(modelId); + ModelResp modelResp = modelService.getModel(modelIds.get(0)); String modelName = modelResp.getName(); List admins = modelService.getModelAdmin(modelResp.getId()); - String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins); + String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelName, admins); throw new InvalidPermissionException(message); } } - public Set getHighSensitiveColsByModelId(Long modelId) { + public Set getHighSensitiveColsByModelId(List modelIds) { Set highSensitiveCols = new HashSet<>(); MetaFilter metaFilter = new MetaFilter(); - metaFilter.setModelIds(Lists.newArrayList(modelId)); + metaFilter.setModelIds(modelIds); metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); List highSensitiveDimensions = dimensionService.getDimensions(metaFilter); List highSensitiveMetrics = metricService.getMetrics(metaFilter); if (!CollectionUtils.isEmpty(highSensitiveDimensions)) { - highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName())); + highSensitiveDimensions.forEach(dim -> highSensitiveCols.add(dim.getBizName())); } if (!CollectionUtils.isEmpty(highSensitiveMetrics)) { - highSensitiveMetrics.stream().forEach(metric -> highSensitiveCols.add(metric.getBizName())); + highSensitiveMetrics.forEach(metric -> highSensitiveCols.add(metric.getBizName())); } return highSensitiveCols; } - public AuthorizedResourceResp getAuthorizedResource(User user, Long modelId, + public AuthorizedResourceResp getAuthorizedResource(User user, List modelIds, Set sensitiveResReq) { List resourceReqList = new ArrayList<>(); - sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelId, res))); + sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res))); QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); queryAuthResReq.setResources(resourceReqList); - queryAuthResReq.setModelId(modelId); + queryAuthResReq.setModelIds(modelIds); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); - log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelId, + log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds, authorizedResource); return authorizedResource; } @@ -124,13 +119,13 @@ public class AuthCommonService { return authService.queryAuthorizedResources(queryAuthResReq, user); } - public Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) { + public Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, List modelIds) { Set resAuthName = new HashSet<>(); List authResGrpList = authorizedResource.getResources(); authResGrpList.stream().forEach(authResGrp -> { List cols = authResGrp.getGroup(); if (!CollectionUtils.isEmpty(cols)) { - cols.stream().filter(col -> modelId.equals(col.getModelId())) + cols.stream().filter(col -> modelIds.contains(col.getModelId())) .forEach(col -> resAuthName.add(col.getName())); } @@ -148,8 +143,9 @@ public class AuthCommonService { } public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns, - Long domainId, AuthorizedResourceResp authResource) { - addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet()); + List modelIds, + AuthorizedResourceResp authResource) { + addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet()); return resultWithColumns; } @@ -245,20 +241,20 @@ public class AuthCommonService { return queryResultWithColumns; } - public void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns, + public void addPromptInfoInfo(List modelIds, QueryResultWithSchemaResp queryResultWithColumns, AuthorizedResourceResp authorizedResource, Set need2Apply) { List filters = authorizedResource.getFilters(); if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) { return; } - List admins = modelService.getModelAdmin(modelId); + List admins = modelService.getModelAdmin(modelIds.get(0)); if (!CollectionUtils.isEmpty(need2Apply)) { String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins); queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo)); } if (!CollectionUtils.isEmpty(filters)) { log.debug("dimensionFilters:{}", filters); - ModelResp modelResp = modelService.getModel(modelId); + ModelResp modelResp = modelService.getModel(modelIds.get(0)); List exprList = new ArrayList<>(); List descList = new ArrayList<>(); filters.stream().forEach(filter -> { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/MaterializationServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/MaterializationServiceImpl.java index f40f4cbe8..50bf56d3d 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/MaterializationServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/MaterializationServiceImpl.java @@ -12,10 +12,13 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.materialization.domain.MaterializationConfService; import com.tencent.supersonic.semantic.model.domain.Catalog; -import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -24,9 +27,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service("MaterializationService") @Slf4j @@ -34,19 +34,17 @@ public class MaterializationServiceImpl implements MaterializationService { protected final MaterializationConfService materializationConfService; protected final ModelService modelService; - protected final DatasourceService datasourceService; protected final Catalog catalog; protected final QueryStructUtils queryStructUtils; protected final QueryService queryService; public MaterializationServiceImpl( MaterializationConfService materializationConfService, - ModelService modelService, DatasourceService datasourceService, + ModelService modelService, Catalog catalog, QueryStructUtils queryStructUtils, QueryService queryService) { this.materializationConfService = materializationConfService; this.modelService = modelService; - this.datasourceService = datasourceService; this.catalog = catalog; this.queryStructUtils = queryStructUtils; this.queryService = queryService; @@ -72,11 +70,12 @@ public class MaterializationServiceImpl implements MaterializationService { ModelSchemaFilterReq modelFilter = new ModelSchemaFilterReq(); modelFilter.setModelIds(Arrays.asList(materializationSourceResp.getModelId())); List modelSchemaRespList = modelService.fetchModelSchema(modelFilter); - List measureRespList = datasourceService.getMeasureListOfModel( + //todo + List measureRespList = modelService.getMeasureListOfModel( Lists.newArrayList(materializationSourceResp.getModelId())); modelSchemaRespList.stream().forEach(m -> { m.getDimensions().stream() - .filter(mm -> mm.getDatasourceId().equals(materializationSourceReq.getDataSourceId()) + .filter(mm -> mm.getModelId().equals(materializationSourceReq.getDataSourceId()) && materializationSourceResp.getDimensions().keySet().contains(mm.getId()) ).forEach(mm -> { dimensionFields.add(mm.getBizName()); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index 2a9f4739b..1511b3093 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -4,42 +4,43 @@ import com.google.common.cache.CacheBuilder; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Cache; -import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; +import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; -import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation; import com.tencent.supersonic.semantic.query.executor.QueryExecutor; import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; +import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation; import com.tencent.supersonic.semantic.query.utils.StatUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Service @@ -75,11 +76,11 @@ public class QueryServiceImpl implements QueryService { @Override @S2SQLPermissionAnnotation @SneakyThrows - public Object queryBySql(QueryS2SQLReq querySqlCmd, User user) { - statUtils.initStatInfo(querySqlCmd, user); + public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) { + statUtils.initStatInfo(queryS2SQLReq, user); QueryStatement queryStatement = new QueryStatement(); try { - queryStatement = convertToQueryStatement(querySqlCmd, user); + queryStatement = convertToQueryStatement(queryS2SQLReq, user); } catch (Exception e) { log.info("convertToQueryStatement has a exception:{}", e.toString()); } @@ -95,18 +96,11 @@ public class QueryServiceImpl implements QueryService { private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - List modelIds = new ArrayList<>(); - modelIds.add(querySqlCmd.getModelId()); - - filter.setModelIds(modelIds); + filter.setModelIds(querySqlCmd.getModelIds()); SchemaService schemaService = ContextUtils.getBean(SchemaService.class); - List domainSchemas = schemaService.fetchModelSchema(filter, user); - ModelSchemaResp domainSchema = null; - if (CollectionUtils.isNotEmpty(domainSchemas)) { - domainSchema = domainSchemas.get(0); - } - QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchema); - queryStatement.setModelId(querySqlCmd.getModelId()); + List modelSchemaResps = schemaService.fetchModelSchema(filter, user); + QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, modelSchemaResps); + queryStatement.setModelIds(querySqlCmd.getModelIds()); return queryStatement; } @@ -116,7 +110,7 @@ public class QueryServiceImpl implements QueryService { log.info("[queryStructCmd:{}]", queryStructCmd); try { statUtils.initStatInfo(queryStructCmd, user); - String cacheKey = cacheUtils.generateCacheKey(queryStructCmd.getModelId().toString(), + String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructCmd.getModelIds()), queryStructCmd.generateCommandMd5()); handleGlobalCacheDisable(queryStructCmd); boolean isCache = isCache(queryStructCmd); @@ -160,7 +154,7 @@ public class QueryServiceImpl implements QueryService { throws Exception { statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user); String cacheKey = cacheUtils.generateCacheKey( - queryMultiStructReq.getQueryStructReqs().get(0).getModelId().toString(), + getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()), queryMultiStructReq.generateCommandMd5()); boolean isCache = isCache(queryMultiStructReq); QueryResultWithSchemaResp queryResultWithColumns; @@ -323,5 +317,9 @@ public class QueryServiceImpl implements QueryService { return queryStructReq; } + private String getKeyByModelIds(List modelIds) { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + } + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java index 69610380b..69e0a34c6 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java @@ -6,11 +6,11 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; import java.util.List; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java index 8bcb1e43c..bfeabe989 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java @@ -1,34 +1,35 @@ package com.tencent.supersonic.semantic.query.service; -import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; - import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; -import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; -import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DomainService; -import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.MetricService; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import com.tencent.supersonic.semantic.model.domain.ModelService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; + @Slf4j @Service public class SchemaServiceImpl implements SchemaService { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java index f72f0c55b..ee04f0c16 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.semantic.query.utils.ComponentFactory; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Slf4j @Component @@ -36,8 +37,8 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine { if (queryExecutor != null) { queryResultWithColumns = queryExecutor.execute(catalog, queryStatement); queryResultWithColumns.setSql(queryStatement.getSql()); - if (queryStatement.getModelId() > 0) { - queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelId()); + if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { + queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds()); } } return queryResultWithColumns; @@ -46,7 +47,7 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine { public QueryStatement plan(QueryStructReq queryStructCmd) throws Exception { QueryStatement queryStatement = queryParser.logicSql(queryStructCmd); queryUtils.checkSqlParse(queryStatement); - queryStatement.setModelId(queryStructCmd.getModelId()); + queryStatement.setModelIds(queryStructCmd.getModelIds()); log.info("queryStatement:{}", queryStatement); return optimize(queryStructCmd, queryStatement); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java index c0dd1f033..1af70e721 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java @@ -12,7 +12,6 @@ import com.tencent.supersonic.semantic.query.parser.calcite.CalciteSqlParser; import com.tencent.supersonic.semantic.query.parser.convert.CalculateAggConverter; import com.tencent.supersonic.semantic.query.parser.convert.DefaultDimValueConverter; import com.tencent.supersonic.semantic.query.parser.convert.MetricCheckConverter; -import com.tencent.supersonic.semantic.query.parser.convert.MultiSourceJoin; import com.tencent.supersonic.semantic.query.parser.convert.ParserDefaultConverter; import java.util.ArrayList; import java.util.HashMap; @@ -83,7 +82,6 @@ public class ComponentFactory { semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); - semanticConverters.add(getBean("MultiSourceJoin", MultiSourceJoin.class)); } private static void initQueryExecutors() { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java index 85ca5607e..be9d626a2 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java @@ -1,35 +1,32 @@ package com.tencent.supersonic.semantic.query.utils; -import static com.tencent.supersonic.common.pojo.Constants.MINUS; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; -import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; -import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; -import com.tencent.supersonic.auth.api.authorization.service.AuthService; -import com.tencent.supersonic.common.pojo.QueryAuthorization; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; -import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; -import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; -import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import java.text.SimpleDateFormat; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import com.tencent.supersonic.semantic.model.domain.DimensionService; +import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; +import com.tencent.supersonic.semantic.query.service.AuthCommonService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -37,40 +34,21 @@ import java.util.Objects; import java.util.Set; import java.util.StringJoiner; import java.util.stream.Collectors; -import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.MetricService; -import com.tencent.supersonic.semantic.model.domain.ModelService; -import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.assertj.core.util.Sets; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.MINUS; @Component @Aspect @Slf4j public class DataPermissionAOP { - - private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat( - new SimpleDateFormat(Constants.DAY_FORMAT)); @Autowired private QueryStructUtils queryStructUtils; @Autowired - private AuthService authService; - @Autowired private DimensionService dimensionService; @Autowired - private MetricService metricService; - @Autowired private ModelService modelService; + @Autowired + private AuthCommonService authCommonService; @Value("${permission.data.enable:true}") private Boolean permissionDataEnable; @@ -93,27 +71,29 @@ public class DataPermissionAOP { throw new RuntimeException("lease provide user information"); } //1. determine whether admin of the model - if (doModelAdmin(user, queryStructReq)) { + if (authCommonService.doModelAdmin(user, queryStructReq.getModelIds())) { return point.proceed(); } // 2. determine whether the subject field is visible - doModelVisible(user, queryStructReq); + authCommonService.doModelVisible(user, queryStructReq.getModelIds()); // 3. fetch data permission meta information - Long modelId = queryStructReq.getModelId(); + List modelIds = queryStructReq.getModelIds(); Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq); - log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege); + log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); - Set sensitiveResByModel = getHighSensitiveColsByModelId(modelId); + Set sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelIds); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq); + log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq); // query user privilege info - AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelId, sensitiveResReq); + AuthorizedResourceResp authorizedResource = authCommonService.getAuthorizedResource(user, + modelIds, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelId()); + Set resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, + queryStructReq.getModelIds()); // 4.if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq); @@ -127,115 +107,20 @@ public class DataPermissionAOP { if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelId, authorizedResource); + return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource); } // 6.if the column has no permission, hit * Set need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req)) .collect(Collectors.toSet()); - QueryResultWithSchemaResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, - need2Apply); - addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply); + QueryResultWithSchemaResp queryResultAfterDesensitization = + authCommonService.desensitizationData(queryResultWithColumns, need2Apply); + authCommonService.addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; } - private boolean doModelAdmin(User user, QueryStructReq queryStructReq) { - Long modelId = queryStructReq.getModelId(); - List modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); - if (CollectionUtils.isEmpty(modelListAdmin)) { - return false; - } else { - Map> id2modelResp = modelListAdmin.stream() - .collect(Collectors.groupingBy(SchemaItem::getId)); - return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId); - } - } - - private void doModelVisible(User user, QueryStructReq queryStructReq) { - Boolean visible = true; - Long modelId = queryStructReq.getModelId(); - List modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE); - if (CollectionUtils.isEmpty(modelListVisible)) { - visible = false; - } else { - Map> id2domainDesc = modelListVisible.stream() - .collect(Collectors.groupingBy(SchemaItem::getId)); - if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) { - visible = false; - } - } - if (!visible) { - ModelResp modelResp = modelService.getModel(modelId); - String modelName = modelResp.getName(); - List admins = modelService.getModelAdmin(modelResp.getId()); - String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins); - throw new InvalidPermissionException(message); - } - - } - - private QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns, - Long domainId, AuthorizedResourceResp authResource) { - addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet()); - return resultWithColumns; - } - - private void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns, - AuthorizedResourceResp authorizedResource, Set need2Apply) { - List filters = authorizedResource.getFilters(); - if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) { - return; - } - List admins = modelService.getModelAdmin(modelId); - if (!CollectionUtils.isEmpty(need2Apply)) { - String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins); - queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo)); - } - if (!CollectionUtils.isEmpty(filters)) { - log.debug("dimensionFilters:{}", filters); - ModelResp modelResp = modelService.getModel(modelId); - List exprList = new ArrayList<>(); - List descList = new ArrayList<>(); - filters.stream().forEach(filter -> { - descList.add(filter.getDescription()); - exprList.add(filter.getExpressions().toString()); - }); - String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s"; - String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins); - - queryResultWithColumns.setQueryAuthorization( - new QueryAuthorization(modelResp.getName(), exprList, descList, message)); - log.info("queryResultWithColumns:{}", queryResultWithColumns); - } - } - - private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception { - QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); - BeanUtils.copyProperties(raw, queryResultWithColumns); - - List columns = new ArrayList<>(); - if (!CollectionUtils.isEmpty(raw.getColumns())) { - String columnsStr = MAPPER.writeValueAsString(raw.getColumns()); - columns = MAPPER.readValue(columnsStr, new TypeReference>() { - }); - queryResultWithColumns.setColumns(columns); - } - queryResultWithColumns.setColumns(columns); - - List> resultData = new ArrayList<>(); - if (!CollectionUtils.isEmpty(raw.getResultList())) { - for (Map line : raw.getResultList()) { - Map newLine = new HashMap<>(); - newLine.putAll(line); - resultData.add(newLine); - } - } - queryResultWithColumns.setResultList(resultData); - return queryResultWithColumns; - } - private boolean allSensitiveResReqIsOk(Set sensitiveResReq, Set resAuthSet) { if (resAuthSet.containsAll(sensitiveResReq)) { return true; @@ -244,50 +129,6 @@ public class DataPermissionAOP { return false; } - private Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) { - Set resAuthName = new HashSet<>(); - List authResGrpList = authorizedResource.getResources(); - authResGrpList.stream().forEach(authResGrp -> { - List cols = authResGrp.getGroup(); - if (!CollectionUtils.isEmpty(cols)) { - cols.stream().filter(col -> modelId.equals(col.getModelId())) - .forEach(col -> resAuthName.add(col.getName())); - } - - }); - log.info("resAuthName:{}", resAuthName); - return resAuthName; - } - - private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId, - Set sensitiveResReq) { - List resourceReqList = new ArrayList<>(); - sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId, res))); - QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); - queryAuthResReq.setResources(resourceReqList); - queryAuthResReq.setModelId(domainId); - AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); - log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId, - authorizedResource); - return authorizedResource; - } - - private Set getHighSensitiveColsByModelId(Long modelId) { - Set highSensitiveCols = new HashSet<>(); - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setModelIds(Lists.newArrayList(modelId)); - metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); - List highSensitiveDimensions = dimensionService.getDimensions(metaFilter); - List highSensitiveMetrics = metricService.getMetrics(metaFilter); - if (!CollectionUtils.isEmpty(highSensitiveDimensions)) { - highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName())); - } - if (!CollectionUtils.isEmpty(highSensitiveMetrics)) { - highSensitiveMetrics.stream().forEach(metric -> highSensitiveCols.add(metric.getBizName())); - } - return highSensitiveCols; - } - private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); @@ -320,64 +161,6 @@ public class DataPermissionAOP { } - private QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set need2Apply) { - log.debug("start desensitizationData logic"); - if (CollectionUtils.isEmpty(need2Apply)) { - log.info("user has all sensitiveRes"); - return raw; - } - - List columns = raw.getColumns(); - - boolean doDesensitization = false; - for (QueryColumn queryColumn : columns) { - if (need2Apply.contains(queryColumn.getNameEn())) { - doDesensitization = true; - break; - } - } - if (!doDesensitization) { - return raw; - } - - QueryResultWithSchemaResp queryResultWithColumns = raw; - try { - queryResultWithColumns = deepCopyResult(raw); - } catch (Exception e) { - log.warn("deepCopyResult: ", e); - } - addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply); - desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply); - return queryResultWithColumns; - } - - private void addAuthorizedSchemaInfo(List columns, Set need2Apply) { - if (CollectionUtils.isEmpty(need2Apply)) { - return; - } - columns.stream().forEach(col -> { - if (need2Apply.contains(col.getNameEn())) { - col.setAuthorized(false); - } - }); - } - - private void desensitizationInternal(List> result, Set need2Apply) { - log.info("start desensitizationInternal logic"); - for (int i = 0; i < result.size(); i++) { - Map row = result.get(i); - Map newRow = new HashMap<>(); - for (String col : row.keySet()) { - if (need2Apply.contains(col)) { - newRow.put(col, "****"); - } else { - newRow.put(col, row.get(col)); - } - } - result.set(i, newRow); - } - } - private void doFilterCheckLogic(QueryStructReq queryStructReq, Set resAuthName, Set sensitiveResReq) { Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq); @@ -385,32 +168,18 @@ public class DataPermissionAOP { .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); Set nameCnSet = new HashSet<>(); - List modelIds = new ArrayList<>(); - modelIds.add(queryStructReq.getModelId()); - List modelInfos = modelService.getModelList(modelIds); - String modelNameCn = Constants.EMPTY; - if (!CollectionUtils.isEmpty(modelInfos)) { - modelNameCn = modelInfos.get(0).getName(); - } - + Map modelRespMap = modelService.getModelMap(); + List modelIds = Lists.newArrayList(queryStructReq.getModelIds()); List dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds)); - String finalDomainNameCn = modelNameCn; dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName())) - .forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName())); + .forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName())); if (!CollectionUtils.isEmpty(need2Apply)) { - ModelResp modelResp = modelInfos.get(0); - List admins = modelService.getModelAdmin(modelResp.getId()); + List admins = modelService.getModelAdmin(modelIds.get(0)); log.info("in doFilterLogic, need2Apply:{}", need2Apply); String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins); throw new InvalidPermissionException(message); } } - - private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) { - log.info("queryAuthResReq:{}", queryAuthResReq); - return authService.queryAuthorizedResources(queryAuthResReq, user); - } - } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java index 050ec0873..c19fe549e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.semantic.query.utils; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; @@ -9,19 +11,9 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Objects; -import java.util.stream.Collectors; - import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -34,6 +26,14 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + @Aspect @Component @Slf4j @@ -56,7 +56,7 @@ public class DimValueAspect { } Object[] args = joinPoint.getArgs(); QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0]; - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelId())); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds())); String sql = queryS2SQLReq.getSql(); log.info("correctorSql before replacing:{}", sql); // if dimensionvalue is alias,consider the true dimensionvalue. @@ -147,8 +147,7 @@ public class DimValueAspect { Object[] args = joinPoint.getArgs(); QueryStructReq queryStructReq = (QueryStructReq) args[0]; - Long modelId = queryStructReq.getModelId(); - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds())); List dimensions = dimensionService.getDimensions(metaFilter); Map> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions); Map> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java index 18a03dea4..1c95f4a81 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java @@ -1,15 +1,11 @@ package com.tencent.supersonic.semantic.query.utils; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; @@ -23,7 +19,6 @@ import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; @@ -33,6 +28,14 @@ import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.service.SchemaService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -46,14 +49,12 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.logging.log4j.util.Strings; -import org.assertj.core.util.Lists; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; @Slf4j @@ -95,7 +96,7 @@ public class QueryStructUtils { private List getDimensionIds(QueryStructReq queryStructCmd) { List dimensionIds = new ArrayList<>(); - List dimensions = catalog.getDimensions(queryStructCmd.getModelId()); + List dimensions = catalog.getDimensions(queryStructCmd.getModelIds()); Map> pair = dimensions.stream() .collect(Collectors.groupingBy(DimensionResp::getBizName)); for (String group : queryStructCmd.getGroups()) { @@ -116,7 +117,7 @@ public class QueryStructUtils { private List getMetricIds(QueryStructReq queryStructCmd) { List metricIds = new ArrayList<>(); - List metrics = catalog.getMetrics(queryStructCmd.getModelId()); + List metrics = catalog.getMetrics(queryStructCmd.getModelIds()); Map> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName)); for (Aggregator agg : queryStructCmd.getAggregators()) { if (pair.containsKey(agg.getColumn())) { @@ -214,7 +215,7 @@ public class QueryStructUtils { Set resNameSet = getResName(queryS2SQLReq); Set resNameEnSet = new HashSet<>(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - List modelIds = Lists.newArrayList(queryS2SQLReq.getModelId()); + List modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds()); filter.setModelIds(modelIds); List modelSchemaRespList = schemaService.fetchModelSchema(filter, user); if (!CollectionUtils.isEmpty(modelSchemaRespList)) { @@ -256,14 +257,16 @@ public class QueryStructUtils { if (CollectionUtils.isEmpty(groups)) { log.warn("group is empty!"); } else { - String group = groups.get(0).equalsIgnoreCase("sys_imp_date") - ? groups.get(1) : groups.get(0); - DimensionResp dimension = catalog.getDimension(group, modelId); - String datasourceBizName = dimension.getDatasourceBizName(); - if (Strings.isNotEmpty(datasourceBizName)) { - internalMetricNamePrefix = datasourceBizName + UNDERLINE; + for (int i = 0; i < groups.size(); i++) { + if (groups.get(i).equalsIgnoreCase("sys_imp_date")) { + continue; + } + DimensionResp dimension = catalog.getDimension(groups.get(i), modelId); + if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getModelBizName())) { + internalMetricNamePrefix = dimension.getModelBizName() + UNDERLINE; + break; + } } - } String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix; return internalMetricName; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java index f215ca041..0ba7e54ee 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java @@ -1,20 +1,24 @@ package com.tencent.supersonic.semantic.query.utils; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; - import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import javax.annotation.PostConstruct; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -26,12 +30,9 @@ import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; @Slf4j @@ -67,9 +68,9 @@ public class QueryUtils { } } - public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long modelId) { - List metricDescList = catalog.getMetrics(modelId); - List dimensionDescList = catalog.getDimensions(modelId); + public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List modelIds) { + List metricDescList = catalog.getMetrics(modelIds); + List dimensionDescList = catalog.getDimensions(modelIds); Map metricRespMap = metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); Map namePair = new HashMap<>(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java index 2e0146091..6d322b0bf 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java @@ -1,8 +1,7 @@ package com.tencent.supersonic.semantic.query.utils; -import static com.tencent.supersonic.common.pojo.Constants.MINUS; - import com.google.common.base.Strings; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; import com.tencent.supersonic.common.pojo.Constants; @@ -15,14 +14,8 @@ import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter; import com.tencent.supersonic.semantic.query.service.AuthCommonService; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.StringJoiner; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -38,6 +31,16 @@ import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import static com.tencent.supersonic.common.pojo.Constants.MINUS; + @Component @Aspect @Order(1) @@ -72,29 +75,29 @@ public class S2SQLDataAspect { if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { throw new RuntimeException("please provide user information"); } - Long modelId = queryS2SQLReq.getModelId(); + List modelIds = queryS2SQLReq.getModelIds(); //1. determine whether admin of the model - if (authCommonService.doModelAdmin(user, modelId)) { + if (authCommonService.doModelAdmin(user, modelIds)) { log.info("determine whether admin of the model!"); return joinPoint.proceed(); } // 2. determine whether the subject field is visible - authCommonService.doModelVisible(user, modelId); + authCommonService.doModelVisible(user, modelIds); // 3. fetch data permission meta information Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user); - log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege); + log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); - Set sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId); + Set sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelIds); Set sensitiveResReq = res4Privilege.parallelStream() .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq); + log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq); // query user privilege info AuthorizedResourceResp authorizedResource = authCommonService - .getAuthorizedResource(user, modelId, sensitiveResReq); + .getAuthorizedResource(user, modelIds, sensitiveResReq); // get sensitiveRes that user has privilege - Set resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId); + Set resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelIds); // 4.if sensitive fields without permission are involved in filter, thrown an exception doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq); @@ -109,7 +112,7 @@ public class S2SQLDataAspect { .allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty log.info("sensitiveResReq is empty"); - return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelId, authorizedResource); + return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource); } // 6.if the column has no permission, hit * @@ -118,7 +121,7 @@ public class S2SQLDataAspect { log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet); QueryResultWithSchemaResp queryResultAfterDesensitization = authCommonService .desensitizationData(queryResultWithColumns, need2Apply); - authCommonService.addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply); + authCommonService.addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; } @@ -163,9 +166,10 @@ public class S2SQLDataAspect { .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); Set nameCnSet = new HashSet<>(); - List modelIds = new ArrayList<>(); - modelIds.add(queryS2SQLReq.getModelId()); - List modelInfos = modelService.getModelList(modelIds); + List modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds()); + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setModelIds(modelIds); + List modelInfos = modelService.getModelList(modelFilter); String modelNameCn = Constants.EMPTY; if (!CollectionUtils.isEmpty(modelInfos)) { modelNameCn = modelInfos.get(0).getName(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java index ba8892c62..aa6364ddf 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.semantic.api.model.enums.QueryOptMode; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.pojo.QueryStat; @@ -17,23 +18,20 @@ import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.model.domain.ModelService; -import com.tencent.supersonic.semantic.api.model.enums.QueryOptMode; - import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + @Component @Slf4j @@ -91,23 +89,20 @@ public class StatUtils { public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); List allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql()); - queryStatInfo.setModelId(queryS2SQLReq.getModelId()); - ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelId()); + queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0)); + ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0)); List dimensions = new ArrayList<>(); - if (Objects.nonNull(modelSchemaResp)) { - dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions()); - } - List metrics = new ArrayList<>(); if (Objects.nonNull(modelSchemaResp)) { + dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions()); metrics = getFieldNames(allFields, modelSchemaResp.getMetrics()); } String userName = getUserName(facadeUser); try { queryStatInfo.setTraceId("") - .setModelId(queryS2SQLReq.getModelId()) + .setModelId(queryS2SQLReq.getModelIds().get(0)) .setUser(userName) .setQueryType(QueryTypeEnum.SQL.getValue()) .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState()) @@ -137,7 +132,7 @@ public class StatUtils { try { queryStatInfo.setTraceId(traceId) - .setModelId(queryStructCmd.getModelId()) + .setModelId(1L) .setUser(user) .setQueryType(QueryTypeEnum.STRUCT.getValue()) .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState()) diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/MaterializationQueryTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/MaterializationQueryTest.java index 982ea5e9e..cb7efac0c 100644 --- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/MaterializationQueryTest.java +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/MaterializationQueryTest.java @@ -10,9 +10,10 @@ import com.tencent.supersonic.semantic.query.parser.QueryParser; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.ComponentFactory; import com.tencent.supersonic.semantic.query.utils.QueryUtils; -import java.util.Arrays; import lombok.extern.slf4j.Slf4j; +import java.util.Arrays; + @Slf4j public class MaterializationQueryTest { @@ -43,7 +44,7 @@ public class MaterializationQueryTest { try { QueryStatement queryStatement = queryParser.logicSql(queryStructReq); queryUtils.checkSqlParse(queryStatement); - queryStatement.setModelId(queryStructReq.getModelId()); + queryStatement.setModelIds(queryStructReq.getModelIds()); log.info("queryStatement:{}", queryStatement); for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { queryOptimizer.rewrite(queryStructReq, queryStatement); diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java index 0ba89345a..c8ff6f6f1 100644 --- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.domain.calcite; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.semantic.api.model.response.SqlParserResp; -import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; +import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl; @@ -53,7 +53,7 @@ class SemanticParserServiceTest { public void test() throws Exception { - DatasourceYamlTpl datasource = new DatasourceYamlTpl(); + DataModelYamlTpl datasource = new DataModelYamlTpl(); datasource.setName("s2_pv_uv_statis"); datasource.setSourceId(1L); datasource.setSqlQuery("SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"); @@ -195,7 +195,7 @@ class SemanticParserServiceTest { private static void addDepartment(SemanticSchema semanticSchema) { - DatasourceYamlTpl datasource = new DatasourceYamlTpl(); + DataModelYamlTpl datasource = new DataModelYamlTpl(); datasource.setName("user_department"); datasource.setSourceId(1L); datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");