From ad203802831dc3a3c4295692fbc1b29df427769c Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Thu, 2 Nov 2023 18:44:58 +0800 Subject: [PATCH] Feature/model data embedding for chat and support status for metric and dimension (#311) * (improvement)(semantic) add offline status for metric and dimension * (improvement)(chat) add metric recall --------- Co-authored-by: jolunoluo --- .../chat/api/pojo/SchemaElement.java | 2 + .../chat/api/pojo/SemanticParseInfo.java | 3 + .../embedding/EmbeddingBasedParser.java | 1 + .../supersonic/chat/plugin/PluginManager.java | 2 +- .../parse/SimilarMetricParseResponder.java | 76 +++++ .../chat/service/impl/ChatServiceImpl.java | 2 - .../chat/service/impl/ConfigServiceImpl.java | 14 +- .../chat/utils/SolvedQueryManager.java | 2 +- .../test/context/MockBeansConfiguration.java | 11 +- .../knowledge/listener/DataAddListener.java | 27 -- .../listener/DataDeleteListener.java | 27 -- .../listener/DataUpdateListener.java | 29 -- .../listener/DictUpdateListener.java | 43 +++ .../common/config}/EmbeddingConfig.java | 2 +- .../supersonic/common/pojo/DataAddEvent.java | 9 +- .../common/pojo/DataDeleteEvent.java | 9 +- .../supersonic/common/pojo/DataEvent.java | 28 ++ .../supersonic/common/pojo/DataItem.java | 26 ++ .../common/pojo/DataUpdateEvent.java | 9 +- .../common/pojo/enums/EventType.java | 9 + .../util/embedding/EmbeddingCollection.java | 17 + .../common/util/embedding/EmbeddingQuery.java | 20 ++ .../common/util/embedding/EmbeddingUtils.java | 107 ++++++ .../common/util/embedding/Retrieval.java | 18 + .../common/util/embedding/RetrieveQuery.java | 20 ++ .../util/embedding/RetrieveQueryResult.java | 15 + .../main/resources/db/semantic-schema-h2.sql | 3 + .../main/resources/META-INF/spring.factories | 3 +- .../src/main/resources/db/schema-h2.sql | 7 +- .../src/main/resources/db/schema-mysql.sql | 9 +- .../src/main/resources/db/sql-update.sql | 9 + .../integration/MetricInterpretTest.java | 2 +- .../integration/MockConfiguration.java | 2 +- .../model/MetricServiceImplTest.java | 36 ++ .../plugin/PluginRecognizeTest.java | 2 +- .../src/test/resources/db/schema-h2.sql | 3 + .../semantic/api/model/pojo/SchemaItem.java | 3 +- .../api/model/request/MetaBatchReq.java | 13 + .../api/model/request/MetricBaseReq.java | 9 + .../semantic/api/model/request/ModelReq.java | 18 + .../utils/MaterializationZipperUtils.java | 6 +- .../model/application/CatalogImpl.java | 11 +- .../application/DatasourceServiceImpl.java | 57 ++-- .../application/DimensionServiceImpl.java | 268 ++++++--------- .../model/application/MetricServiceImpl.java | 313 +++++++++--------- .../model/application/ModelServiceImpl.java | 63 ++-- .../application/ViewInfoServiceImpl.java | 8 +- .../model/domain/DatasourceService.java | 7 +- .../model/domain/DimensionService.java | 19 +- .../semantic/model/domain/MetricService.java | 29 +- .../semantic/model/domain/ModelService.java | 3 +- .../model/domain/dataobject/DatasourceDO.java | 71 ++-- .../dataobject/DatasourceDOExample.java | 145 ++++++-- .../model/domain/dataobject/ModelDO.java | 42 +++ .../domain/dataobject/ModelDOExample.java | 130 ++++++++ .../listener/MetaEmbeddingListener.java | 56 ++++ .../model/domain/pojo/MetaFilter.java | 9 + .../semantic/model/domain/pojo/Metric.java | 9 - .../repository/DatasourceRepository.java | 2 - .../repository/DimensionRepository.java | 15 +- .../domain/repository/MetricRepository.java | 13 +- .../domain/utils/DatasourceConverter.java | 6 +- .../domain/utils/DimensionConverter.java | 41 +-- .../model/domain/utils/MetricConverter.java | 50 ++- .../model/domain/utils/ModelConvert.java | 28 +- .../mapper/DimensionDOCustomMapper.java | 2 + .../mapper/MetricDOCustomMapper.java | 2 + .../repository/DatasourceRepositoryImpl.java | 17 +- .../repository/DimensionRepositoryImpl.java | 57 +--- .../repository/MetricRepositoryImpl.java | 45 +-- .../repository/ModelRepositoryImpl.java | 11 +- .../model/rest/DatasourceController.java | 23 +- .../model/rest/DimensionController.java | 29 +- .../semantic/model/rest/MetricController.java | 30 +- .../semantic/model/rest/ModelController.java | 7 +- .../resources/mapper/DatasourceDOMapper.xml | 44 ++- .../resources/mapper/DimensionDOMapper.xml | 2 +- .../main/resources/mapper/MetricDOMapper.xml | 2 +- .../main/resources/mapper/ModelDOMapper.xml | 64 +++- .../mapper/custom/DimensionDOCustomMapper.xml | 21 +- .../mapper/custom/MetricDOCustomMapper.xml | 24 +- .../DatabaseApplicationServiceTest.java | 0 .../DatasourceApplicationServiceTest.java | 0 .../DomainApplicationServiceTest.java | 0 .../query/service/AuthCommonService.java | 10 +- .../query/service/SchemaServiceImpl.java | 4 +- .../query/utils/DataPermissionAOP.java | 12 +- .../semantic/query/utils/DimValueAspect.java | 9 +- .../semantic/query/utils/S2QLDataAspect.java | 5 +- 89 files changed, 1572 insertions(+), 896 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SimilarMetricParseResponder.java delete mode 100644 chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataAddListener.java delete mode 100644 chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java delete mode 100644 chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java create mode 100644 chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java rename {chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding => common/src/main/java/com/tencent/supersonic/common/config}/EmbeddingConfig.java (93%) create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingCollection.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingQuery.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingUtils.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/Retrieval.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQuery.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQueryResult.java create mode 100644 launchers/standalone/src/test/java/com/tencent/supersonic/integration/model/MetricServiceImplTest.java create mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetaBatchReq.java create mode 100644 semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/listener/MetaEmbeddingListener.java delete mode 100644 semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatabaseApplicationServiceTest.java delete mode 100644 semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceApplicationServiceTest.java delete mode 100644 semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DomainApplicationServiceTest.java diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java index 2f6f86cb1..97a43dda6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java @@ -28,6 +28,8 @@ public class SchemaElement implements Serializable { private String defaultAgg; + private int order; + @Override public boolean equals(Object o) { if (this == o) { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 010e777bb..f4dd3bb03 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -53,6 +53,9 @@ public class SemanticParseInfo { @Override public int compare(SchemaElement o1, SchemaElement o2) { + if (o1.getOrder() != o2.getOrder()) { + return o1.getOrder() - o2.getOrder(); + } int len1 = o1.getName().length(); int len2 = o2.getName().length(); if (len1 != len2) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java index b6d6f8507..403ad5b4c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.parser.plugin.PluginParser; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginRecallResult; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.util.ContextUtils; import java.util.Comparator; import java.util.List; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java index 0c8429bfd..8fcb9da4a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.PluginTool; -import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp; import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval; import com.tencent.supersonic.chat.plugin.event.PluginAddEvent; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SimilarMetricParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SimilarMetricParseResponder.java new file mode 100644 index 000000000..547fa8c95 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SimilarMetricParseResponder.java @@ -0,0 +1,76 @@ +package com.tencent.supersonic.chat.responder.parse; + +import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; +import com.tencent.supersonic.chat.query.QueryManager; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.embedding.EmbeddingUtils; +import com.tencent.supersonic.common.util.embedding.Retrieval; +import com.tencent.supersonic.common.util.embedding.RetrieveQuery; +import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult; +import com.tencent.supersonic.semantic.model.domain.listener.MetaEmbeddingListener; +import org.springframework.util.CollectionUtils; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class SimilarMetricParseResponder implements ParseResponder { + + + @Override + public void fillResponse(ParseResp parseResp, QueryContext queryContext, List chatParseDOS) { + if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) { + return; + } + fillSimilarMetric(parseResp.getSelectedParses().iterator().next()); + } + + private void fillSimilarMetric(SemanticParseInfo parseInfo) { + if (!QueryManager.isMetricQuery(parseInfo.getQueryMode()) + || CollectionUtils.isEmpty(parseInfo.getMetrics())) { + return; + } + List metricNames = parseInfo.getMetrics().stream() + .map(SchemaElement::getName).collect(Collectors.toList()); + Map filterCondition = new HashMap<>(); + filterCondition.put("modelId", parseInfo.getModelId().toString()); + filterCondition.put("type", SchemaElementType.METRIC.name()); + RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames) + .filterCondition(filterCondition).queryEmbeddings(null).build(); + EmbeddingUtils embeddingUtils = ContextUtils.getBean(EmbeddingUtils.class); + List retrieveQueryResults = embeddingUtils.retrieveQuery( + MetaEmbeddingListener.COLLECTION_NAME, retrieveQuery, 10); + if (CollectionUtils.isEmpty(retrieveQueryResults)) { + return; + } + List retrievals = retrieveQueryResults.stream() + .flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream()) + .sorted(Comparator.comparingDouble(Retrieval::getDistance).reversed()) + .distinct().collect(Collectors.toList()); + Set metricIds = parseInfo.getMetrics().stream().map(SchemaElement::getId).collect(Collectors.toSet()); + int metricOrder = 0; + for (SchemaElement metric : parseInfo.getMetrics()) { + metric.setOrder(metricOrder++); + } + for (Retrieval retrieval : retrievals) { + if (!metricIds.contains(retrieval.getId())) { + SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(retrieval.getMetadata()), + SchemaElement.class); + if (retrieval.getMetadata().containsKey("modelId")) { + schemaElement.setModel(Long.parseLong(retrieval.getMetadata().get("modelId"))); + } + schemaElement.setOrder(metricOrder++); + parseInfo.getMetrics().add(schemaElement); + } + } + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java index 4be4d902c..455fff0b7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java @@ -18,7 +18,6 @@ import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.persistence.repository.ChatRepository; - import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Comparator; @@ -29,7 +28,6 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; - import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.utils.SolvedQueryManager; import com.tencent.supersonic.common.util.JsonUtil; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java index b3d6a8460..ad13f7279 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.service.impl; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ModelSchema; @@ -38,10 +39,12 @@ import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; +import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -143,16 +146,19 @@ public class ConfigServiceImpl implements ConfigService { List filterMetricIdList = blackMetricIdList.stream().distinct().collect(Collectors.toList()); ItemNameVisibilityInfo itemNameVisibility = new ItemNameVisibilityInfo(); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); if (!CollectionUtils.isEmpty(blackDimIdList)) { - List dimensionRespList = dimensionService.getDimensions(modelId); + List dimensionRespList = dimensionService.getDimensions(metaFilter); List blackDimNameList = dimensionRespList.stream().filter(o -> filterDimIdList.contains(o.getId())) - .map(o -> o.getName()).collect(Collectors.toList()); + .map(SchemaItem::getName).collect(Collectors.toList()); itemNameVisibility.setBlackDimNameList(blackDimNameList); } if (!CollectionUtils.isEmpty(blackMetricIdList)) { - List metricRespList = metricService.getMetrics(modelId); + + List metricRespList = metricService.getMetrics(metaFilter); List blackMetricList = metricRespList.stream().filter(o -> filterMetricIdList.contains(o.getId())) - .map(o -> o.getName()).collect(Collectors.toList()); + .map(SchemaItem::getName).collect(Collectors.toList()); itemNameVisibility.setBlackMetricNameList(blackMetricList); } return itemNameVisibility; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java index decdaeb80..45af0f6b5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java @@ -5,7 +5,7 @@ import com.alibaba.fastjson.serializer.SerializerFeature; import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; -import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp; import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval; import lombok.extern.slf4j.Slf4j; diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java index 11699ffef..d30873460 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.test.context; -import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.when; @@ -26,6 +26,9 @@ import com.tencent.supersonic.semantic.model.domain.MetricService; import java.util.ArrayList; import java.util.Arrays; import java.util.List; + +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import org.mockito.Mockito; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -81,11 +84,11 @@ public class MockBeansConfiguration { } public static void dimensionDescBuild(DimensionService dimensionService, List dimensionDescs) { - when(dimensionService.getDimensions(anyList())).thenReturn(dimensionDescs); + when(dimensionService.getDimensions(any(DimensionFilter.class))).thenReturn(dimensionDescs); } - public static void metricDescBuild(MetricService dimensionService, List metricDescs) { - when(dimensionService.getMetrics(anyList())).thenReturn(metricDescs); + public static void metricDescBuild(MetricService metricService, List metricDescs) { + when(metricService.getMetrics(any(MetaFilter.class))).thenReturn(metricDescs); } public static DimSchemaResp getDimensionDesc(Long id, String bizName, String name) { diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataAddListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataAddListener.java deleted file mode 100644 index c358b6508..000000000 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataAddListener.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.knowledge.listener; - -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DataAddEvent; -import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.common.pojo.enums.DictWordType; -import com.tencent.supersonic.knowledge.utils.HanlpHelper; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class DataAddListener implements ApplicationListener { - @Override - public void onApplicationEvent(DataAddEvent event) { - DictWord dictWord = new DictWord(); - dictWord.setWord(event.getName()); - String sign = DictWordType.NATURE_SPILT; - String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); - String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; - dictWord.setNature(nature); - dictWord.setNatureWithFrequency(natureWithFrequency); - log.info("dataAddListener begins to add data:{}", dictWord); - HanlpHelper.addToCustomDictionary(dictWord); - } -} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java deleted file mode 100644 index 9d812e348..000000000 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.knowledge.listener; - -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DataDeleteEvent; -import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.common.pojo.enums.DictWordType; -import com.tencent.supersonic.knowledge.utils.HanlpHelper; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class DataDeleteListener implements ApplicationListener { - @Override - public void onApplicationEvent(DataDeleteEvent event) { - DictWord dictWord = new DictWord(); - dictWord.setWord(event.getName()); - String sign = DictWordType.NATURE_SPILT; - String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); - String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; - dictWord.setNature(nature); - dictWord.setNatureWithFrequency(natureWithFrequency); - log.info("dataDeleteListener begins to delete data:{}", dictWord); - HanlpHelper.removeFromCustomDictionary(dictWord); - } -} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java deleted file mode 100644 index 13db47355..000000000 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tencent.supersonic.knowledge.listener; - -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DataUpdateEvent; -import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.common.pojo.enums.DictWordType; -import com.tencent.supersonic.knowledge.utils.HanlpHelper; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class DataUpdateListener implements ApplicationListener { - @Override - public void onApplicationEvent(DataUpdateEvent event) { - DictWord dictWord = new DictWord(); - dictWord.setWord(event.getName()); - String sign = DictWordType.NATURE_SPILT; - String nature = sign + event.getModelId() + sign + event.getId() + event.getType(); - String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; - dictWord.setNature(nature); - dictWord.setNatureWithFrequency(natureWithFrequency); - log.info("dataUpdateListener begins to update data:{}", dictWord); - HanlpHelper.removeFromCustomDictionary(dictWord); - dictWord.setWord(event.getNewName()); - HanlpHelper.addToCustomDictionary(dictWord); - } -} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java new file mode 100644 index 000000000..97cf47f4b --- /dev/null +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java @@ -0,0 +1,43 @@ +package com.tencent.supersonic.knowledge.listener; + +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.common.pojo.enums.EventType; +import com.tencent.supersonic.knowledge.dictionary.DictWord; +import com.tencent.supersonic.knowledge.utils.HanlpHelper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +@Component +@Slf4j +public class DictUpdateListener implements ApplicationListener { + + @Override + public void onApplicationEvent(DataEvent dataEvent) { + if (CollectionUtils.isEmpty(dataEvent.getDataItems())) { + return; + } + dataEvent.getDataItems().forEach(dataItem -> { + DictWord dictWord = new DictWord(); + dictWord.setWord(dataItem.getName()); + String sign = DictWordType.NATURE_SPILT; + String nature = sign + dataItem.getModelId() + sign + dataItem.getId() + + sign + dataItem.getType().getName(); + String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY; + dictWord.setNature(nature); + dictWord.setNatureWithFrequency(natureWithFrequency); + if (EventType.ADD.equals(dataEvent.getEventType())) { + HanlpHelper.addToCustomDictionary(dictWord); + } else if (EventType.DELETE.equals(dataEvent.getEventType())) { + HanlpHelper.removeFromCustomDictionary(dictWord); + } else if (EventType.UPDATE.equals(dataEvent.getEventType())) { + HanlpHelper.removeFromCustomDictionary(dictWord); + dictWord.setWord(dataItem.getNewName()); + HanlpHelper.addToCustomDictionary(dictWord); + } + }); + } +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java similarity index 93% rename from chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java rename to common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java index 46ff9f848..6bd1483f6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.parser.plugin.embedding; +package com.tencent.supersonic.common.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java index 07d100c8a..3b1cc2279 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.pojo; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import org.springframework.context.ApplicationEvent; public class DataAddEvent extends ApplicationEvent { @@ -7,9 +8,9 @@ public class DataAddEvent extends ApplicationEvent { private String name; private Long modelId; private Long id; - private String type; + private TypeEnums type; - public DataAddEvent(Object source, String name, Long modelId, Long id, String type) { + public DataAddEvent(Object source, String name, Long modelId, Long id, TypeEnums type) { super(source); this.name = name; this.modelId = modelId; @@ -41,11 +42,11 @@ public class DataAddEvent extends ApplicationEvent { this.modelId = modelId; } - public void setType(String type) { + public void setType(TypeEnums type) { this.type = type; } - public String getType() { + public TypeEnums getType() { return type; } } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java index 83bf309fe..febb48b63 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.pojo; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import org.springframework.context.ApplicationEvent; public class DataDeleteEvent extends ApplicationEvent { @@ -7,9 +8,9 @@ public class DataDeleteEvent extends ApplicationEvent { private String name; private Long modelId; private Long id; - private String type; + private TypeEnums type; - public DataDeleteEvent(Object source, String name, Long modelId, Long id, String type) { + public DataDeleteEvent(Object source, String name, Long modelId, Long id, TypeEnums type) { super(source); this.name = name; this.modelId = modelId; @@ -41,11 +42,11 @@ public class DataDeleteEvent extends ApplicationEvent { this.modelId = modelId; } - public void setType(String type) { + public void setType(TypeEnums type) { this.type = type; } - public String getType() { + public TypeEnums getType() { return type; } } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java new file mode 100644 index 000000000..92a865f07 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java @@ -0,0 +1,28 @@ +package com.tencent.supersonic.common.pojo; + + +import com.tencent.supersonic.common.pojo.enums.EventType; +import org.springframework.context.ApplicationEvent; +import java.util.List; + + +public class DataEvent extends ApplicationEvent { + + private List dataItems; + + private EventType eventType; + + public DataEvent(Object source, List dataItems, EventType eventType) { + super(source); + this.dataItems = dataItems; + this.eventType = eventType; + } + + public List getDataItems() { + return dataItems; + } + + public EventType getEventType() { + return eventType; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java new file mode 100644 index 000000000..beb48e0a4 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java @@ -0,0 +1,26 @@ +package com.tencent.supersonic.common.pojo; + +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class DataItem { + + private Long id; + + private String bizName; + + private String name; + + private String newName; + + private TypeEnums type; + + private Long modelId; + + public String getNewName() { + return newName == null ? name : newName; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java index 908c28f09..74c61e390 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.pojo; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import org.springframework.context.ApplicationEvent; public class DataUpdateEvent extends ApplicationEvent { @@ -8,9 +9,9 @@ public class DataUpdateEvent extends ApplicationEvent { private String newName; private Long modelId; private Long id; - private String type; + private TypeEnums type; - public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, String type) { + public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, TypeEnums type) { super(source); this.name = name; this.newName = newName; @@ -51,11 +52,11 @@ public class DataUpdateEvent extends ApplicationEvent { this.modelId = modelId; } - public void setType(String type) { + public void setType(TypeEnums type) { this.type = type; } - public String getType() { + public TypeEnums getType() { return type; } } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java new file mode 100644 index 000000000..f91ceca6c --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java @@ -0,0 +1,9 @@ +package com.tencent.supersonic.common.pojo.enums; + +public enum EventType { + + ADD, + UPDATE, + DELETE + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingCollection.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingCollection.java new file mode 100644 index 000000000..9277e7ae5 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingCollection.java @@ -0,0 +1,17 @@ +package com.tencent.supersonic.common.util.embedding; + + +import lombok.Data; + +import java.util.Map; + +@Data +public class EmbeddingCollection { + + private String id; + + private String name; + + private Map metaData; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingQuery.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingQuery.java new file mode 100644 index 000000000..46d6e17fe --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingQuery.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.common.util.embedding; + + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class EmbeddingQuery { + + private String queryId; + + private String query; + + private Map metadata; + + private List queryEmbedding; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingUtils.java new file mode 100644 index 000000000..d186e9c9c --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingUtils.java @@ -0,0 +1,107 @@ +package com.tencent.supersonic.common.util.embedding; + +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.config.EmbeddingConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class EmbeddingUtils { + + @Autowired + private EmbeddingConfig embeddingConfig; + + private RestTemplate restTemplate = new RestTemplate(); + + public void addCollection(String collectionName) { + List collections = getCollectionList(); + if (collections.contains(collectionName)) { + return; + } + String url = String.format("%s/create_collection?collection_name=%s", + embeddingConfig.getUrl(), collectionName); + doRequest(url, null, HttpMethod.GET); + } + + public void addQuery(String collectionName, List queries) { + if (CollectionUtils.isEmpty(queries)) { + return; + } + String url = String.format("%s/add_query?collection_name=%s", + embeddingConfig.getUrl(), collectionName); + doRequest(url, JSONObject.toJSONString(queries, SerializerFeature.WriteMapNullValue), HttpMethod.POST); + } + + public void deleteQuery(String collectionName, List queries) { + if (CollectionUtils.isEmpty(queries)) { + return; + } + List queryIds = queries.stream().map(EmbeddingQuery::getQueryId).collect(Collectors.toList()); + String url = String.format("%s/delete_query_by_ids?collection_name=%s", + embeddingConfig.getUrl(), collectionName); + doRequest(url, JSONObject.toJSONString(queryIds), HttpMethod.POST); + } + + public List retrieveQuery(String collectionName, RetrieveQuery retrieveQuery, int num) { + String url = String.format("%s/retrieve_query?collection_name=%s&n_results=%s", + embeddingConfig.getUrl(), collectionName, num); + ResponseEntity responseEntity = doRequest(url, JSONObject.toJSONString(retrieveQuery, + SerializerFeature.WriteMapNullValue), HttpMethod.POST); + if (!responseEntity.hasBody()) { + return Lists.newArrayList(); + } + return JSONObject.parseArray(responseEntity.getBody(), RetrieveQueryResult.class); + } + + private List getCollectionList() { + String url = embeddingConfig.getUrl() + "/list_collections"; + ResponseEntity responseEntity = doRequest(url, null, HttpMethod.GET); + if (!responseEntity.hasBody()) { + return Lists.newArrayList(); + } + List embeddingCollections = JSONObject.parseArray(responseEntity.getBody(), + EmbeddingCollection.class); + return embeddingCollections.stream().map(EmbeddingCollection::getName).collect(Collectors.toList()); + } + + private ResponseEntity doRequest(String url, String jsonBody, HttpMethod httpMethod) { + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setLocation(URI.create(url)); + URI requestUrl = UriComponentsBuilder + .fromHttpUrl(url).build().encode().toUri(); + HttpEntity entity = new HttpEntity<>(headers); + if (jsonBody != null) { + log.info("[embedding] request body :{}", jsonBody); + entity = new HttpEntity<>(jsonBody, headers); + } + ResponseEntity responseEntity = restTemplate.exchange(requestUrl, + httpMethod, entity, new ParameterizedTypeReference() {}); + log.info("[embedding] url :{} result body:{}", url, responseEntity); + return responseEntity; + } catch (Throwable e) { + log.warn("connect to embedding service failed, url:{}", url); + } + return ResponseEntity.of(Optional.empty()); + } + +} + diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/Retrieval.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/Retrieval.java new file mode 100644 index 000000000..8d2e82195 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/Retrieval.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.common.util.embedding; + +import lombok.Data; + +import java.util.Map; + +@Data +public class Retrieval { + + private Long id; + + private double distance; + + private String query; + + private Map metadata; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQuery.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQuery.java new file mode 100644 index 000000000..737792fa5 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQuery.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.common.util.embedding; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +public class RetrieveQuery { + + private List queryTextsList; + + private Map filterCondition; + + private List> queryEmbeddings; + + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQueryResult.java b/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQueryResult.java new file mode 100644 index 000000000..8991f1497 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQueryResult.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.common.util.embedding; + + +import lombok.Data; + +import java.util.List; + +@Data +public class RetrieveQueryResult { + + private String query; + + private List retrieval; + +} diff --git a/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql b/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql index 375bdd5f8..08114a5b6 100644 --- a/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql +++ b/launchers/semantic/src/main/resources/db/semantic-schema-h2.sql @@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID `alias` varchar(255) DEFAULT NULL , -- internal name + `status` INT DEFAULT NULL, -- 0 is off the shelf, 1 is normal + `description` varchar(500) DEFAULT NULL , `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , @@ -66,6 +68,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` ( `description` varchar(500) DEFAULT NULL , `database_id` INT NOT NULL , `datasource_detail` LONGVARCHAR NOT NULL , + `status` int(11) DEFAULT NULL , `created_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , `updated_at` TIMESTAMP NOT NULL , diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 1f71cbdfd..3eb0ffc0a 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -41,7 +41,8 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.responder.parse.ParseResponder=\ com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \ - com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder + com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \ + com.tencent.supersonic.chat.responder.parse.SimilarMetricParseResponder com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\ com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 1cbe237b6..f1bfebd9e 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -119,6 +119,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID `alias` varchar(255) DEFAULT NULL , -- internal name + `status` INT DEFAULT NULL, + `description` varchar(500) DEFAULT NULL , `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , @@ -160,6 +162,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` ( `description` varchar(500) DEFAULT NULL , `database_id` INT NOT NULL , `datasource_detail` LONGVARCHAR NOT NULL , + `status` int(11) DEFAULT NULL , `depends` varchar(500) DEFAULT NULL , `created_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , @@ -182,7 +185,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` ( `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) DEFAULT NULL , - `status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted + `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal `sensitive_level` INT NOT NULL , `type` varchar(50) NOT NULL , -- type proxy,expr `type_params` LONGVARCHAR DEFAULT NULL , @@ -207,7 +210,7 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` ( `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) NOT NULL , - `status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted + `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal `sensitive_level` INT DEFAULT NULL , `data_type` varchar(50) DEFAULT NULL , -- type date,array,varchar `type` varchar(50) NOT NULL , -- type categorical,time diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index bcb54e345..8603144a3 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -141,6 +141,7 @@ CREATE TABLE `s2_datasource` ( `description` varchar(500) DEFAULT NULL COMMENT '数据源描述', `database_id` bigint(20) NOT NULL COMMENT '数据库实例ID', `datasource_detail` mediumtext NOT NULL COMMENT '数据源配置', + `status` int(11) DEFAULT NULL , `depends` text DEFAULT NULL COMMENT '上游依赖标识', `created_at` datetime NOT NULL COMMENT '创建时间', `created_by` varchar(100) NOT NULL COMMENT '创建人', @@ -202,7 +203,7 @@ CREATE TABLE `s2_dimension` ( `name` varchar(255) NOT NULL COMMENT '维度名称', `biz_name` varchar(255) NOT NULL COMMENT '字段名称', `description` varchar(500) NOT NULL COMMENT '描述', - `status` int(10) NOT NULL COMMENT '维度状态,0正常,1下架,2删除', + `status` int(10) NOT NULL COMMENT '维度状态,0正常,1下架', `sensitive_level` int(10) DEFAULT NULL COMMENT '敏感级别', `type` varchar(50) NOT NULL COMMENT '维度类型 categorical,time', `type_params` text COMMENT '类型参数', @@ -245,9 +246,9 @@ CREATE TABLE `s2_metric` ( `name` varchar(255) NOT NULL COMMENT '指标名称', `biz_name` varchar(255) NOT NULL COMMENT '字段名称', `description` varchar(500) DEFAULT NULL COMMENT '描述', - `status` int(10) NOT NULL COMMENT '指标状态,0正常,1下架,2删除', + `status` int(10) NOT NULL COMMENT '指标状态,0未启用,1启用', `sensitive_level` int(10) NOT NULL COMMENT '敏感级别', - `type` varchar(50) NOT NULL COMMENT '指标类型 proxy,expr', + `type` varchar(50) NOT NULL COMMENT '指标类型', `type_params` text NOT NULL COMMENT '类型参数', `created_at` datetime NOT NULL COMMENT '创建时间', `created_by` varchar(100) NOT NULL COMMENT '创建人', @@ -268,6 +269,8 @@ CREATE TABLE `s2_model` ( `biz_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, `domain_id` bigint(20) DEFAULT NULL, `alias` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL, + `status` int(11) DEFAULT NULL, + `description` varchar(500) DEFAULT NULL, `viewer` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL, `view_org` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL, `admin` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL, diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 9b45c40e0..4e2f31638 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -68,3 +68,12 @@ alter table s2_datasource add column depends text COMMENT '上游依赖标识' a --20231018 UPDATE `s2_agent` SET `config` = replace (`config`,'DSL','LLM_S2QL') WHERE `config` LIKE '%DSL%'; + +--20231023 +alter table s2_model add column status int null after alias; +alter table s2_model add column description varchar(500) null after status; +alter table s2_datasource add column status int null after database_id; +update s2_model set status = 1; +update s2_datasource set status = 1; +update s2_metric set status = 1; +update s2_dimension set status = 1; \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java index a5df68ec6..446274f07 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java @@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.StandaloneLauncher; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.query.llm.interpret.LLmAnswerResp; import com.tencent.supersonic.chat.service.AgentService; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MockConfiguration.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MockConfiguration.java index 79ba8b6d8..09a8aa349 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MockConfiguration.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MockConfiguration.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.integration; import com.google.common.collect.Lists; -import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp; import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval; import com.tencent.supersonic.chat.plugin.PluginManager; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/model/MetricServiceImplTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/model/MetricServiceImplTest.java new file mode 100644 index 000000000..c74321ca2 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/model/MetricServiceImplTest.java @@ -0,0 +1,36 @@ +package com.tencent.supersonic.integration.model; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.StandaloneLauncher; +import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; +import com.tencent.supersonic.semantic.api.model.enums.MetricTypeEnum; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.model.domain.MetricService; +import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import java.util.List; + +@SpringBootTest(classes = StandaloneLauncher.class) +@ActiveProfiles("local") +public class MetricServiceImplTest { + + @Autowired + protected MetricService metricService; + + @Test + void getMetrics() { + MetricFilter metricFilter = new MetricFilter(); + metricFilter.setType(MetricTypeEnum.ATOMIC.name()); + metricFilter.setModelIds(Lists.newArrayList(1L)); + metricFilter.setSensitiveLevel(SensitiveLevelEnum.LOW.ordinal()); + List metricResps = metricService.getMetrics(metricFilter); + Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> metricResp.getModelId().equals(2L))); + Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> + metricResp.getSensitiveLevel().equals(SensitiveLevelEnum.HIGH.ordinal()))); + } + +} \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java index 5142e48c2..b7e81836a 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; +import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.QueryService; diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index b6e0a758c..7cb42165a 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -134,6 +134,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` ( `biz_name` varchar(255) DEFAULT NULL , -- internal name `domain_id` INT DEFAULT '0' , -- parent domain ID `alias` varchar(255) DEFAULT NULL , -- alias name + `status` INT DEFAULT NULL , + `description` varchar(500) DEFAULT NULL , `created_at` TIMESTAMP DEFAULT NULL , `created_by` varchar(100) DEFAULT NULL , `updated_at` TIMESTAMP DEFAULT NULL , @@ -176,6 +178,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` ( `database_id` INT NOT NULL , `depends` varchar(500) DEFAULT NULL , `datasource_detail` LONGVARCHAR NOT NULL , + `status` int(11) DEFAULT NULL , `created_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , `updated_at` TIMESTAMP NOT NULL , diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java index ef09f8693..9e98d9eb3 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.semantic.api.model.pojo; import com.google.common.base.Objects; import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import java.util.ArrayList; import java.util.Arrays; @@ -25,7 +24,7 @@ public class SchemaItem extends RecordInfo { private String description; - private Integer status = StatusEnum.ONLINE.getCode(); + private Integer status; private TypeEnums typeEnum; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetaBatchReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetaBatchReq.java new file mode 100644 index 000000000..e90e9ed16 --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetaBatchReq.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.semantic.api.model.request; + +import lombok.Data; +import java.util.List; + +@Data +public class MetaBatchReq { + + private List ids; + + private Integer status; + +} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetricBaseReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetricBaseReq.java index c47c11f15..dd371906a 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetricBaseReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetricBaseReq.java @@ -4,6 +4,8 @@ import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.common.pojo.DataFormat; import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; import java.util.List; @@ -22,4 +24,11 @@ public class MetricBaseReq extends SchemaItem { private RelateDimension relateDimension; + public String getTag() { + if (CollectionUtils.isEmpty(tags)) { + return ""; + } + return StringUtils.join(tags, ","); + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java index 12f087c51..0d82d83a1 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java @@ -30,4 +30,22 @@ public class ModelReq extends SchemaItem { private Entity entity; private List drillDownDimensions; + + public String getViewer() { + return String.join(",", viewers); + } + + public String getViewOrg() { + return String.join(",", viewOrgs); + } + + + public String getAdmin() { + return String.join(",", admins); + } + + public String getAdminOrg() { + return String.join(",", adminOrgs); + } + } diff --git a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java index d68ba2b97..c2f1b374e 100644 --- a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java +++ b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.semantic.materialization.domain.utils; +import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum; @@ -9,6 +10,7 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat import com.tencent.supersonic.semantic.api.materialization.response.MaterializationResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.model.domain.DimensionService; +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -60,7 +62,9 @@ public class MaterializationZipperUtils implements MaterializationUtils { return ""; } StringJoiner joiner = new StringJoiner(","); - Map dimIdAndDim = dimensionService.getDimensions(materializationResp.getModelId()) + DimensionFilter dimensionFilter = new DimensionFilter(); + dimensionFilter.setModelIds(Lists.newArrayList(materializationResp.getModelId())); + Map dimIdAndDim = dimensionService.getDimensions(dimensionFilter) .stream().collect(Collectors.toMap(DimensionResp::getId, value -> value, (v1, v2) -> v2)); materializationElementRespList.stream() .filter(element -> TypeEnums.DIMENSION.equals(element.getType()) && ElementFrequencyEnum.LOW.equals( diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java index 7b0fd39fb..d1f733883 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.semantic.model.application; +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; @@ -22,6 +24,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; + +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -84,7 +88,9 @@ public class CatalogImpl implements Catalog { @Override public List getDimensions(Long modelId) { - return dimensionService.getDimensions(modelId); + MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId)); + metricFilter.setStatus(StatusEnum.ONLINE.getCode()); + return dimensionService.getDimensions(metricFilter); } @Override @@ -94,7 +100,8 @@ public class CatalogImpl implements Catalog { @Override public List getMetrics(Long modelId) { - return metricService.getMetrics(modelId); + MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId)); + return metricService.getMetrics(metricFilter); } @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java index da8fe79b8..2f2702206 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.semantic.model.application; -import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; @@ -36,6 +36,7 @@ import com.tencent.supersonic.semantic.model.domain.manager.DatasourceYamlManage import com.tencent.supersonic.semantic.model.domain.manager.DimensionYamlManager; import com.tencent.supersonic.semantic.model.domain.manager.MetricYamlManager; import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository; import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository; import com.tencent.supersonic.semantic.model.domain.utils.DatasourceConverter; @@ -74,7 +75,6 @@ public class DatasourceServiceImpl implements DatasourceService { private DateInfoRepository dateInfoRepository; - public DatasourceServiceImpl(DatasourceRepository datasourceRepository, DatabaseService databaseService, @Lazy DimensionService dimensionService, @@ -89,36 +89,25 @@ public class DatasourceServiceImpl implements DatasourceService { @Override @Transactional - public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception { - preCheck(datasourceReq); + public Datasource createDatasource(DatasourceReq datasourceReq, User user) throws Exception { + checkName(datasourceReq); checkExist(datasourceReq); Datasource datasource = DatasourceConverter.convert(datasourceReq); - log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource)); saveDatasource(datasource, user); - Optional datasourceDescOptional = getDatasource(datasourceReq.getModelId(), - datasourceReq.getBizName()); - if (!datasourceDescOptional.isPresent()) { - throw new RuntimeException("创建数据源失败"); - } - DatasourceResp datasourceResp = datasourceDescOptional.get(); - datasource.setId(datasourceResp.getId()); batchCreateDimension(datasource, user); batchCreateMetric(datasource, user); - return datasourceResp; + return datasource; } @Override @Transactional - public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception { - preCheck(datasourceReq); + public Datasource updateDatasource(DatasourceReq datasourceReq, User user) throws Exception { + checkName(datasourceReq); Datasource datasource = DatasourceConverter.convert(datasourceReq); - - log.info("[update datasource] object:{}", JSONObject.toJSONString(datasource)); - + updateDatasource(datasource, user); batchCreateDimension(datasource, user); batchCreateMetric(datasource, user); - DatasourceDO datasourceDO = updateDatasource(datasource, user); - return DatasourceConverter.convert(datasourceDO); + return datasource; } private DatasourceDO updateDatasource(Datasource datasource, User user) { @@ -168,7 +157,6 @@ public class DatasourceServiceImpl implements DatasourceService { return measureResps; } - private void batchCreateDimension(Datasource datasource, User user) throws Exception { List dimensionReqs = DatasourceConverter.convertDimensionList(datasource); dimensionService.createDimensionBatch(dimensionReqs, user); @@ -179,7 +167,6 @@ public class DatasourceServiceImpl implements DatasourceService { metricService.createMetricBatch(exprMetricReqs, user); } - private Optional getDatasource(Long modelId, String bizName) { List datasourceResps = getDatasourceList(modelId); if (CollectionUtils.isEmpty(datasourceResps)) { @@ -196,13 +183,11 @@ public class DatasourceServiceImpl implements DatasourceService { //保存并获取自增ID private void saveDatasource(Datasource datasource, User user) { DatasourceDO datasourceDO = DatasourceConverter.convert(datasource, user); - log.info("[save datasource] datasourceDO:{}", JSONObject.toJSONString(datasourceDO)); datasourceRepository.createDatasource(datasourceDO); datasource.setId(datasourceDO.getId()); } - - private void preCheck(DatasourceReq datasourceReq) { + private void checkName(DatasourceReq datasourceReq) { if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) { String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName()); throw new InvalidArgumentException(message); @@ -297,18 +282,23 @@ public class DatasourceServiceImpl implements DatasourceService { @Override - public void deleteDatasource(Long id) { + public void deleteDatasource(Long id, User user) { DatasourceDO datasourceDO = datasourceRepository.getDatasourceById(id); if (datasourceDO == null) { return; } - checkDelete(datasourceDO.getModelId(), id); - datasourceRepository.deleteDatasource(id); + checkDelete(id); + datasourceDO.setStatus(StatusEnum.DELETED.getCode()); + datasourceDO.setUpdatedAt(new Date()); + datasourceDO.setUpdatedBy(user.getName()); + datasourceRepository.updateDatasource(datasourceDO); } - private void checkDelete(Long modelId, Long datasourceId) { - List metricResps = metricService.getMetrics(modelId, datasourceId); - List dimensionResps = dimensionService.getDimensionsByDatasource(datasourceId); + private void checkDelete(Long datasourceId) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setDatasourceId(datasourceId); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) { throw new RuntimeException("存在基于该数据源创建的指标和维度, 暂不能删除, 请确认"); } @@ -420,11 +410,12 @@ public class DatasourceServiceImpl implements DatasourceService { List datasourceYamlTplList, List metricYamlTplList) { for (Long modelId : modelIds) { List datasourceResps = getDatasourceList(modelId); - List metricResps = metricService.getMetrics(modelId); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); + List metricResps = metricService.getMetrics(metaFilter); metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(MetricConverter.metricInfo2Metric(metricResps))); Long databaseId = datasourceResps.iterator().next().getDatabaseId(); DatabaseResp databaseResp = databaseService.getDatabase(databaseId); - List dimensionResps = dimensionService.getDimensions(modelId); + List dimensionResps = dimensionService.getDimensions(metaFilter); for (DatasourceResp datasourceResp : datasourceResps) { datasourceYamlTplList.add(DatasourceYamlManager.convert2YamlObj( DatasourceConverter.datasourceInfo2Datasource(datasourceResp), databaseResp)); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 944ccae67..9c37a70ae 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -7,37 +7,37 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DataAddEvent; -import com.tencent.supersonic.common.pojo.DataDeleteEvent; -import com.tencent.supersonic.common.pojo.DataUpdateEvent; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.common.pojo.DataItem; +import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.enums.EventType; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; -import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.model.domain.DatabaseService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository; import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter; import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; -import com.tencent.supersonic.semantic.model.domain.pojo.Dimension; import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; - +import java.util.Date; import java.util.List; import java.util.Map; import java.util.ArrayList; import java.util.HashMap; import java.util.stream.Collectors; - import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -63,7 +63,7 @@ public class DimensionServiceImpl implements DimensionService { private DatabaseService databaseService; @Autowired - private ApplicationEventPublisher applicationEventPublisher; + private ApplicationEventPublisher eventPublisher; public DimensionServiceImpl(DimensionRepository dimensionRepository, @@ -76,23 +76,17 @@ public class DimensionServiceImpl implements DimensionService { this.datasourceService = datasourceService; this.chatGptHelper = chatGptHelper; this.databaseService = databaseService; - } @Override public void createDimension(DimensionReq dimensionReq, User user) { checkExist(Lists.newArrayList(dimensionReq)); - Dimension dimension = DimensionConverter.convert(dimensionReq); - log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension)); - dimension.createdBy(user.getName()); - saveDimension(dimension); - String type = DictWordType.DIMENSION.getType(); - DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId()); - applicationEventPublisher.publishEvent( - new DataAddEvent(this, dimension.getName(), dimension.getModelId(), dimensionResp.getId(), type)); + dimensionReq.createdBy(user.getName()); + DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimensionReq); + dimensionRepository.createDimension(dimensionDO); + sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD); } - @Override public void createDimensionBatch(List dimensionReqs, User user) { if (CollectionUtils.isEmpty(dimensionReqs)) { @@ -100,44 +94,79 @@ public class DimensionServiceImpl implements DimensionService { } Long modelId = dimensionReqs.get(0).getModelId(); List dimensionResps = getDimensions(modelId); - Map dimensionRespMap = dimensionResps.stream() + Map bizNameMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); - List dimensions = dimensionReqs.stream().map(DimensionConverter::convert) + Map nameMap = dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); + List dimensionToInsert = dimensionReqs.stream() + .filter(dimension -> !bizNameMap.containsKey(dimension.getBizName()) + && !nameMap.containsKey(dimension.getName())) .collect(Collectors.toList()); - List dimensionToInsert = dimensions.stream() - .filter(dimension -> !dimensionRespMap.containsKey(dimension.getBizName())) + if (CollectionUtils.isEmpty(dimensionToInsert)) { + return; + } + List dimensionDOS = dimensionToInsert.stream().peek(dimension -> + dimension.createdBy(user.getName())) + .map(DimensionConverter::convert2DimensionDO) .collect(Collectors.toList()); - - log.info("[create dimension] object:{}", JSONObject.toJSONString(dimensions)); - saveDimensionBatch(dimensionToInsert, user); + dimensionRepository.createDimensionBatch(dimensionDOS); + sendEventBatch(dimensionDOS, EventType.ADD); } @Override public void updateDimension(DimensionReq dimensionReq, User user) { - if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) { - throw new InvalidArgumentException("名称包含特殊字符, 请修改"); - } - Dimension dimension = DimensionConverter.convert(dimensionReq); - dimension.updatedBy(user.getName()); - log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension)); - updateDimension(dimension); - DimensionResp dimensionResp = getDimension(dimensionReq.getId()); - //动态更新字典 - String type = DictWordType.DIMENSION.getType(); - if (dimensionResp != null) { - applicationEventPublisher.publishEvent( - new DataUpdateEvent(this, dimensionResp.getName(), - dimensionReq.getName(), - dimension.getModelId(), - dimensionResp.getId(), type)); + checkExist(Lists.newArrayList(dimensionReq)); + DimensionDO dimensionDO = dimensionRepository.getDimensionById(dimensionReq.getId()); + dimensionReq.updatedBy(user.getName()); + String oldName = dimensionDO.getName(); + DimensionConverter.convert(dimensionDO, dimensionReq); + dimensionRepository.updateDimension(dimensionDO); + if (!oldName.equals(dimensionDO.getName())) { + sendEvent(DataItem.builder().modelId(dimensionDO.getModelId()).newName(dimensionReq.getName()) + .name(oldName).type(TypeEnums.DIMENSION) + .id(dimensionDO.getId()).build(), EventType.UPDATE); } } - protected void updateDimension(Dimension dimension) { - DimensionDO dimensionDO = dimensionRepository.getDimensionById(dimension.getId()); - dimensionRepository.updateDimension(DimensionConverter.convert(dimensionDO, dimension)); + @Override + public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) { + if (CollectionUtils.isEmpty(metaBatchReq.getIds())) { + return; + } + DimensionFilter dimensionFilter = new DimensionFilter(); + dimensionFilter.setIds(metaBatchReq.getIds()); + List dimensionDOS = dimensionRepository.getDimension(dimensionFilter); + if (CollectionUtils.isEmpty(dimensionDOS)) { + return; + } + dimensionDOS = dimensionDOS.stream() + .peek(dimensionDO -> { + dimensionDO.setStatus(metaBatchReq.getStatus()); + dimensionDO.setUpdatedAt(new Date()); + dimensionDO.setUpdatedBy(user.getName()); + }) + .collect(Collectors.toList()); + dimensionRepository.batchUpdateStatus(dimensionDOS); + if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus()) + || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) { + sendEventBatch(dimensionDOS, EventType.DELETE); + } else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) { + sendEventBatch(dimensionDOS, EventType.ADD); + } } + @Override + public void deleteDimension(Long id, User user) { + DimensionDO dimensionDO = dimensionRepository.getDimensionById(id); + if (dimensionDO == null) { + throw new RuntimeException(String.format("the dimension %s not exist", id)); + } + dimensionDO.setStatus(StatusEnum.DELETED.getCode()); + dimensionDO.setUpdatedAt(new Date()); + dimensionDO.setUpdatedBy(user.getName()); + dimensionRepository.updateDimension(dimensionDO); + sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE); + } @Override public DimensionResp getDimension(String bizName, Long modelId) { @@ -177,48 +206,15 @@ public class DimensionServiceImpl implements DimensionService { return dimensionRepository.getDimension(dimensionFilter); } - @Override - public List getDimensions(List ids) { - List dimensionResps = Lists.newArrayList(); - List dimensionDOS = dimensionRepository.getDimensionListByIds(ids); - Map modelFullPathMap = modelService.getModelFullPathMap(); - if (!CollectionUtils.isEmpty(dimensionDOS)) { - dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelFullPathMap, - new HashMap<>())) - .collect(Collectors.toList()); - } - return dimensionResps; + public List getDimensions(MetaFilter metaFilter) { + DimensionFilter dimensionFilter = new DimensionFilter(); + BeanUtils.copyProperties(metaFilter, dimensionFilter); + return convertList(dimensionRepository.getDimension(dimensionFilter), datasourceService.getDatasourceMap()); } - @Override - public List getDimensions(Long modelId) { - return convertList(getDimensionDOS(modelId), datasourceService.getDatasourceMap()); - } - - @Override - public List getDimensions() { - return convertList(getDimensionDOS(), datasourceService.getDatasourceMap()); - } - - @Override - public List getDimensionsByModelIds(List modelIds) { - List dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds); - return convertList(dimensionDOS, datasourceService.getDatasourceMap()); - } - - @Override - public List getDimensionsByDatasource(Long datasourceId) { - List dimensionResps = Lists.newArrayList(); - List dimensionDOS = dimensionRepository.getDimensionListOfDatasource(datasourceId); - if (!CollectionUtils.isEmpty(dimensionDOS)) { - dimensionResps = dimensionDOS.stream() - .map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), - new HashMap<>())) - .collect(Collectors.toList()); - } - return dimensionResps; + private List getDimensions(Long modelId) { + return getDimensions(new MetaFilter(Lists.newArrayList(modelId))); } private List convertList(List dimensionDOS, @@ -234,75 +230,6 @@ public class DimensionServiceImpl implements DimensionService { return dimensionResps; } - - @Override - public List getHighSensitiveDimension(Long modelId) { - List dimensionResps = getDimensions(modelId); - if (CollectionUtils.isEmpty(dimensionResps)) { - return dimensionResps; - } - return dimensionResps.stream() - .filter(dimensionResp -> SensitiveLevelEnum.HIGH.getCode().equals(dimensionResp.getSensitiveLevel())) - .collect(Collectors.toList()); - } - - - protected List getDimensionDOS(Long modelId) { - return dimensionRepository.getDimensionListOfmodel(modelId); - } - - protected List getDimensionDOS() { - return dimensionRepository.getDimensionList(); - } - - - @Override - public List getAllHighSensitiveDimension() { - List dimensionResps = Lists.newArrayList(); - List dimensionDOS = dimensionRepository.getAllDimensionList(); - if (CollectionUtils.isEmpty(dimensionDOS)) { - return dimensionResps; - } - return convertList(dimensionDOS.stream() - .filter(dimensionDO -> SensitiveLevelEnum.HIGH.getCode().equals(dimensionDO.getSensitiveLevel())) - .collect(Collectors.toList()), new HashMap<>()); - } - - - public void saveDimension(Dimension dimension) { - DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimension); - log.info("[save dimension] dimensionDO:{}", JSONObject.toJSONString(dimensionDO)); - dimensionRepository.createDimension(dimensionDO); - dimension.setId(dimensionDO.getId()); - } - - private void saveDimensionBatch(List dimensions, User user) { - if (CollectionUtils.isEmpty(dimensions)) { - return; - } - dimensions = dimensions.stream().peek(dimension -> dimension.createdBy(user.getName())) - .collect(Collectors.toList()); - List dimensionDOS = dimensions.stream() - .map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList()); - log.info("[save dimension] dimensionDO:{}", JSONObject.toJSONString(dimensionDOS)); - dimensionRepository.createDimensionBatch(dimensionDOS); - } - - - @Override - public void deleteDimension(Long id) { - DimensionDO dimensionDO = dimensionRepository.getDimensionById(id); - if (dimensionDO == null) { - throw new RuntimeException(String.format("the dimension %s not exist", id)); - } - dimensionRepository.deleteDimension(id); - //动态更新字典 - String type = DictWordType.DIMENSION.getType(); - applicationEventPublisher.publishEvent( - new DataDeleteEvent(this, dimensionDO.getName(), dimensionDO.getModelId(), dimensionDO.getId(), type)); - - } - @Override public List mockAlias(DimensionReq dimensionReq, String mockType, User user) { String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(), @@ -368,20 +295,43 @@ public class DimensionServiceImpl implements DimensionService { private void checkExist(List dimensionReqs) { Long modelId = dimensionReqs.get(0).getModelId(); List dimensionResps = getDimensions(modelId); + Map bizNameMap = dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); for (DimensionReq dimensionReq : dimensionReqs) { if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) { throw new InvalidArgumentException("名称包含特殊字符, 请修改"); } - for (DimensionResp dimensionResp : dimensionResps) { - if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) { - throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName())); + if (bizNameMap.containsKey(dimensionReq.getBizName())) { + DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName()); + if (!dimensionResp.getId().equals(dimensionReq.getId())) { + throw new RuntimeException(String.format("该模型下存在相同的维度字段名:%s 创建人:%s", + dimensionReq.getBizName(), dimensionResp.getCreatedBy())); } - if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) { - throw new RuntimeException( - String.format("存在相同的维度名: %s", dimensionReq.getBizName())); + } + if (nameMap.containsKey(dimensionReq.getName())) { + DimensionResp dimensionResp = nameMap.get(dimensionReq.getName()); + if (!dimensionResp.getId().equals(dimensionReq.getId())) { + throw new RuntimeException(String.format("该模型下存在相同的维度名:%s 创建人:%s", + dimensionReq.getName(), dimensionResp.getCreatedBy())); } } } } + private void sendEventBatch(List dimensionDOS, EventType eventType) { + List dataItems = dimensionDOS.stream().map(dimensionDO -> + DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName()) + .modelId(dimensionDO.getModelId()).type(TypeEnums.DIMENSION).build()) + .collect(Collectors.toList()); + eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType)); + } + + private void sendEvent(DataItem dataItem, EventType eventType) { + eventPublisher.publishEvent(new DataEvent(this, + Lists.newArrayList(dataItem), eventType)); + } + + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index af8d67858..f9ff7469d 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -6,31 +6,32 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DataAddEvent; -import com.tencent.supersonic.common.pojo.DataDeleteEvent; -import com.tencent.supersonic.common.pojo.DataUpdateEvent; +import com.tencent.supersonic.common.pojo.DataItem; +import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.common.pojo.enums.EventType; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository; import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; import com.tencent.supersonic.semantic.model.domain.MetricService; -import com.tencent.supersonic.semantic.model.domain.pojo.Metric; - +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -60,7 +61,7 @@ public class MetricServiceImpl implements MetricService { private ChatGptHelper chatGptHelper; @Autowired - private ApplicationEventPublisher applicationEventPublisher; + private ApplicationEventPublisher eventPublisher; public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, @@ -73,17 +74,13 @@ public class MetricServiceImpl implements MetricService { } @Override - public void creatExprMetric(MetricReq metricReq, User user) { - check(Lists.newArrayList(metricReq)); - Metric metric = MetricConverter.convert(metricReq); - metric.createdBy(user.getName()); - log.info("[create metric] object:{}", JSONObject.toJSONString(metric)); - saveMetric(metric); - //动态更新字典 - String type = DictWordType.METRIC.getType(); - MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName()); - applicationEventPublisher.publishEvent( - new DataAddEvent(this, metric.getName(), metric.getModelId(), metricResp.getId(), type)); + public void createMetric(MetricReq metricReq, User user) { + checkExist(Lists.newArrayList(metricReq)); + checkParam(metricReq); + metricReq.createdBy(user.getName()); + MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq); + metricRepository.createMetric(metricDO); + sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD); } @Override @@ -91,47 +88,79 @@ public class MetricServiceImpl implements MetricService { if (CollectionUtils.isEmpty(metricReqs)) { return; } - List metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList()); Long modelId = metricReqs.get(0).getModelId(); - List metricResps = getMetricByModelId(modelId); - Map metricRespMap = metricResps.stream() + List metricResps = getMetricInSameDomain(modelId); + Map bizNameMap = metricResps.stream() .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); - List metricToInsert = metrics.stream() - .filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList()); - log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert)); - saveMetricBatch(metricToInsert, user); + Map nameMap = metricResps.stream() + .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); + List metricToInsert = metricReqs.stream() + .filter(metric -> !bizNameMap.containsKey(metric.getBizName()) + && !nameMap.containsKey(metric.getName())).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(metricToInsert)) { + return; + } + List metricDOS = metricToInsert.stream().peek(metric -> metric.createdBy(user.getName())) + .map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); + metricRepository.createMetricBatch(metricDOS); + sendEventBatch(metricDOS, EventType.ADD); } @Override - public List getMetrics(Long modelId) { - return convertList(metricRepository.getMetricList(modelId)); + public void updateExprMetric(MetricReq metricReq, User user) { + checkParam(metricReq); + checkExist(Lists.newArrayList(metricReq)); + metricReq.updatedBy(user.getName()); + MetricDO metricDO = metricRepository.getMetricById(metricReq.getId()); + String oldName = metricDO.getName(); + MetricConverter.convert(metricDO, metricReq); + metricRepository.updateMetric(metricDO); + if (!oldName.equals(metricDO.getName())) { + DataItem dataItem = getDataItem(metricDO); + dataItem.setName(oldName); + dataItem.setNewName(metricDO.getName()); + sendEvent(getDataItem(metricDO), EventType.UPDATE); + } } @Override - public List getMetrics() { - return convertList(metricRepository.getMetricList()); + public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) { + if (CollectionUtils.isEmpty(metaBatchReq.getIds())) { + return; + } + MetricFilter metricFilter = new MetricFilter(); + metricFilter.setIds(metaBatchReq.getIds()); + List metricDOS = metricRepository.getMetric(metricFilter); + if (CollectionUtils.isEmpty(metricDOS)) { + return; + } + metricDOS = metricDOS.stream() + .peek(metricDO -> { + metricDO.setStatus(metaBatchReq.getStatus()); + metricDO.setUpdatedAt(new Date()); + metricDO.setUpdatedBy(user.getName()); + }) + .collect(Collectors.toList()); + metricRepository.batchUpdateStatus(metricDOS); + if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus()) + || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) { + sendEventBatch(metricDOS, EventType.DELETE); + } else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) { + sendEventBatch(metricDOS, EventType.ADD); + } } @Override - public List getMetrics(Long modelId, Long datasourceId) { - List metricResps = convertList(metricRepository.getMetricList(modelId)); - return metricResps.stream().filter(metricResp -> { - Set datasourceIdSet = metricResp.getTypeParams().getMeasures().stream() - .map(Measure::getDatasourceId) - .filter(Objects::nonNull).collect(Collectors.toSet()); - return !CollectionUtils.isEmpty(datasourceIdSet) && datasourceIdSet.contains(datasourceId); - }).collect(Collectors.toList()); - } - - public List getMetrics(List ids) { - List metricDOS = metricRepository.getMetricListByIds(ids); - return convertList(metricDOS); - } - - @Override - public List getMetricsByModelIds(List modelIds) { - List metricDOS = metricRepository.getMetricList(modelIds); - return convertList(metricDOS); + public void deleteMetric(Long id, User user) { + MetricDO metricDO = metricRepository.getMetricById(id); + if (metricDO == null) { + throw new RuntimeException(String.format("the metric %s not exist", id)); + } + metricDO.setStatus(StatusEnum.DELETED.getCode()); + metricDO.setUpdatedAt(new Date()); + metricDO.setUpdatedBy(user.getName()); + metricRepository.updateMetric(metricDO); + sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE); } @Override @@ -155,10 +184,30 @@ public class MetricServiceImpl implements MetricService { return pageInfo; } - private List queryMetric(MetricFilter metricFilter) { + protected List queryMetric(MetricFilter metricFilter) { return metricRepository.getMetric(metricFilter); } + @Override + public List getMetrics(MetaFilter metaFilter) { + MetricFilter metricFilter = new MetricFilter(); + BeanUtils.copyProperties(metaFilter, metricFilter); + List metricResps = convertList(queryMetric(metricFilter)); + if (metricFilter.getDatasourceId() != null) { + return filterByDatasource(metricFilter.getDatasourceId(), metricResps); + } + return metricResps; + } + + private List filterByDatasource(Long datasourceId, List metricResps) { + return metricResps.stream().filter(metricResp -> { + Set datasourceIdSet = metricResp.getTypeParams().getMeasures().stream() + .map(Measure::getDatasourceId) + .filter(Objects::nonNull).collect(Collectors.toSet()); + return !CollectionUtils.isEmpty(datasourceIdSet) + && datasourceIdSet.contains(datasourceId); + }).collect(Collectors.toList()); + } private void fillAdminRes(List metricResps, User user) { List modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); @@ -171,22 +220,19 @@ public class MetricServiceImpl implements MetricService { metricResp.setHasAdminRes(true); } } - } @Override public MetricResp getMetric(Long modelId, String bizName) { - List metricResps = getMetricByModelId(modelId); + MetricFilter metricFilter = new MetricFilter(); + metricFilter.setBizName(bizName); + metricFilter.setModelIds(Lists.newArrayList(modelId)); + List metricResps = getMetrics(metricFilter); MetricResp metricResp = null; if (CollectionUtils.isEmpty(metricResps)) { return metricResp; } - for (MetricResp metric : metricResps) { - if (metric.getBizName().equalsIgnoreCase(bizName)) { - metricResp = metric; - } - } - return metricResp; + return metricResps.get(0); } private MetricResp getMetric(Long id) { @@ -197,87 +243,6 @@ public class MetricServiceImpl implements MetricService { return MetricConverter.convert2MetricResp(metricDO, new HashMap<>()); } - @Override - public void updateExprMetric(MetricReq metricReq, User user) { - preCheckMetric(metricReq); - Metric metric = MetricConverter.convert(metricReq); - metric.updatedBy(user.getName()); - log.info("[update metric] object:{}", JSONObject.toJSONString(metric)); - List metricRespList = getMetrics(metricReq.getModelId()).stream().filter( - o -> o.getId().equals(metricReq.getId())).collect(Collectors.toList()); - updateMetric(metric); - //动态更新字典 - String type = DictWordType.METRIC.getType(); - //MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName()); - if (!CollectionUtils.isEmpty(metricRespList)) { - log.info("metricRespList size:{}", metricRespList.size()); - log.info("name:{}", metricRespList.get(0).getName()); - applicationEventPublisher.publishEvent( - new DataUpdateEvent(this, metricRespList.get(0).getName(), - metricReq.getName(), - metric.getModelId(), - metricRespList.get(0).getId(), type)); - } - } - - public void saveMetric(Metric metric) { - MetricDO metricDO = MetricConverter.convert2MetricDO(metric); - log.info("[save metric] metricDO:{}", JSONObject.toJSONString(metricDO)); - metricRepository.createMetric(metricDO); - metric.setId(metricDO.getId()); - } - - protected void updateMetric(Metric metric) { - MetricDO metricDO = metricRepository.getMetricById(metric.getId()); - metricRepository.updateMetric(MetricConverter.convert(metricDO, metric)); - } - - public List getMetricByModelId(Long modelId) { - return convertList(getMetricDOByModelId(modelId)); - } - - protected List getMetricDOByModelId(Long modelId) { - List metricDOS = metricRepository.getAllMetricList(); - return metricDOS.stream().filter(metricDO -> Objects.equals(metricDO.getModelId(), modelId)) - .collect(Collectors.toList()); - } - - @Override - public List getHighSensitiveMetric(Long modelId) { - List metricResps = getMetricByModelId(modelId); - if (CollectionUtils.isEmpty(metricResps)) { - return metricResps; - } - return metricResps.stream() - .filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel())) - .collect(Collectors.toList()); - } - - @Override - public List getAllHighSensitiveMetric() { - List metricResps = Lists.newArrayList(); - List metricDOS = metricRepository.getAllMetricList(); - if (CollectionUtils.isEmpty(metricDOS)) { - return metricResps; - } - return convertList(metricDOS.stream() - .filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel())) - .collect(Collectors.toList())); - } - - @Override - public void deleteMetric(Long id) { - MetricDO metricDO = metricRepository.getMetricById(id); - if (metricDO == null) { - throw new RuntimeException(String.format("the metric %s not exist", id)); - } - metricRepository.deleteMetric(id); - //动态更新字典 - String type = DictWordType.METRIC.getType(); - applicationEventPublisher.publishEvent( - new DataDeleteEvent(this, metricDO.getName(), metricDO.getModelId(), metricDO.getId(), type)); - } - @Override public List mockAlias(MetricReq metricReq, String mockType, User user) { @@ -289,7 +254,7 @@ public class MetricServiceImpl implements MetricService { @Override public Set getMetricTags() { - List metricResps = getMetrics(); + List metricResps = getMetrics(new MetaFilter()); if (CollectionUtils.isEmpty(metricResps)) { return new HashSet<>(); } @@ -311,17 +276,7 @@ public class MetricServiceImpl implements MetricService { return modelResp.getDrillDownDimensions(); } - private void saveMetricBatch(List metrics, User user) { - if (CollectionUtils.isEmpty(metrics)) { - return; - } - List metricDOS = metrics.stream().peek(metric -> metric.createdBy(user.getName())) - .map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); - log.info("[save metric] metrics:{}", JSONObject.toJSONString(metricDOS)); - metricRepository.createMetricBatch(metricDOS); - } - - private void preCheckMetric(MetricReq metricReq) { + private void checkParam(MetricReq metricReq) { MetricTypeParams typeParams = metricReq.getTypeParams(); List measures = typeParams.getMeasures(); if (CollectionUtils.isEmpty(measures)) { @@ -335,22 +290,42 @@ public class MetricServiceImpl implements MetricService { } } - private void check(List exprMetricReqList) { - Long modelId = exprMetricReqList.get(0).getModelId(); - List metricResps = getMetrics(modelId); - for (MetricReq exprMetricReq : exprMetricReqList) { - for (MetricResp metricResp : metricResps) { - if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) { - throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName())); + private void checkExist(List metricReqs) { + Long modelId = metricReqs.get(0).getModelId(); + List metricResps = getMetricInSameDomain(modelId); + Map bizNameMap = metricResps.stream() + .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); + Map nameMap = metricResps.stream() + .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); + for (MetricReq metricReq : metricReqs) { + if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) { + throw new InvalidArgumentException("名称包含特殊字符, 请修改"); + } + if (bizNameMap.containsKey(metricReq.getBizName())) { + MetricResp metricResp = bizNameMap.get(metricReq.getBizName()); + if (!metricResp.getId().equals(metricReq.getId())) { + throw new RuntimeException(String.format("该主题域下存在相同的指标字段名:%s 创建人:%s", + metricReq.getBizName(), metricResp.getCreatedBy())); } - if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) { - throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName())); + } + if (nameMap.containsKey(metricReq.getName())) { + MetricResp metricResp = nameMap.get(metricReq.getName()); + if (!metricResp.getId().equals(metricReq.getId())) { + throw new RuntimeException(String.format("该主题域下存在相同的指标名:%s 创建人:%s", + metricReq.getName(), metricResp.getCreatedBy())); } - preCheckMetric(exprMetricReq); } } } + private List getMetricInSameDomain(Long modelId) { + ModelResp modelResp = modelService.getModel(modelId); + Long domainId = modelResp.getDomainId(); + List modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId)); + List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); + return getMetrics(new MetaFilter(modelIds)); + } + private List convertList(List metricDOS) { List metricResps = Lists.newArrayList(); Map modelMap = modelService.getModelMap(); @@ -362,5 +337,21 @@ public class MetricServiceImpl implements MetricService { return metricResps; } + private void sendEventBatch(List metricDOS, EventType eventType) { + List dataItems = metricDOS.stream().map(this::getDataItem) + .collect(Collectors.toList()); + eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType)); + } + + private void sendEvent(DataItem dataItem, EventType eventType) { + eventPublisher.publishEvent(new DataEvent(this, + Lists.newArrayList(dataItem), eventType)); + } + + private DataItem getDataItem(MetricDO metricDO) { + return DataItem.builder().id(metricDO.getId()).name(metricDO.getName()) + .bizName(metricDO.getBizName()) + .modelId(metricDO.getModelId()).type(TypeEnums.METRIC).build(); + } } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 8dbe43511..3ae4975b3 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.semantic.model.application; -import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; @@ -28,7 +28,9 @@ import com.tencent.supersonic.semantic.model.domain.DomainService; import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; -import com.tencent.supersonic.semantic.model.domain.pojo.Model; +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; import com.tencent.supersonic.semantic.model.domain.utils.ModelConvert; import java.util.ArrayList; @@ -56,7 +58,6 @@ public class ModelServiceImpl implements ModelService { private final DomainService domainService; private final UserService userService; private final DatabaseService databaseService; - private final Catalog catalog; public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService, @@ -76,30 +77,33 @@ public class ModelServiceImpl implements ModelService { @Override public void createModel(ModelReq modelReq, User user) { - log.info("[create model] req : {}", JSONObject.toJSONString(modelReq)); - Model model = ModelConvert.convert(modelReq); - log.info("[create model] object:{}", JSONObject.toJSONString(modelReq)); - saveModel(model, user); + modelReq.createdBy(user.getName()); + ModelDO modelDO = ModelConvert.convert(modelReq); + modelRepository.createModel(modelDO); } @Override public void updateModel(ModelReq modelReq, User user) { ModelDO modelDO = getModelDO(modelReq.getId()); - modelDO.setUpdatedAt(new Date()); - modelDO.setUpdatedBy(user.getName()); + modelReq.updatedBy(user.getName()); BeanMapper.mapper(modelReq, modelDO); - modelDO.setAdmin(String.join(",", modelReq.getAdmins())); - modelDO.setAdminOrg(String.join(",", modelReq.getAdminOrgs())); - modelDO.setViewer(String.join(",", modelReq.getViewers())); - modelDO.setViewOrg(String.join(",", modelReq.getViewOrgs())); - modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); + if (modelReq.getEntity() != null) { + modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); + } + if (modelReq.getDrillDownDimensions() != null) { + modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions())); + } modelRepository.updateModel(modelDO); } @Override - public void deleteModel(Long id) { + public void deleteModel(Long id, User user) { checkDelete(id); - modelRepository.deleteModel(id); + ModelDO modelDO = getModelDO(id); + modelDO.setStatus(StatusEnum.DELETED.getCode()); + modelDO.setUpdatedAt(new Date()); + modelDO.setUpdatedBy(user.getName()); + modelRepository.updateModel(modelDO); } @Override @@ -178,8 +182,9 @@ public class ModelServiceImpl implements ModelService { } private void checkDelete(Long id) { - List metricResps = metricService.getMetrics(id); - List dimensionResps = dimensionService.getDimensions(id); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(id)); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); List datasourceResps = datasourceService.getDatasourceList(id); if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(datasourceResps) || !CollectionUtils.isEmpty(dimensionResps)) { @@ -187,12 +192,6 @@ public class ModelServiceImpl implements ModelService { } } - private void saveModel(Model model, User user) { - ModelDO modelDO = ModelConvert.convert(model, user); - modelRepository.createModel(modelDO); - model.setId(modelDO.getId()); - } - private List convertList(List modelDOS) { List modelResps = Lists.newArrayList(); if (CollectionUtils.isEmpty(modelDOS)) { @@ -209,9 +208,9 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelResps)) { return modelResps; } - Map> metricMap = metricService.getMetrics().stream() + Map> metricMap = metricService.getMetrics(new MetricFilter()).stream() .collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionMap = dimensionService.getDimensions().stream() + Map> dimensionMap = dimensionService.getDimensions(new DimensionFilter()).stream() .collect(Collectors.groupingBy(DimensionResp::getModelId)); modelResps.forEach(modelResp -> { modelResp.setDimensionCnt(dimensionMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size()); @@ -280,15 +279,17 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelIds)) { modelIds = generateModelIdsReq(modelSchemaFilterReq); } - Map> metricRespMap = metricService.getMetricsByModelIds(modelIds) + MetaFilter metaFilter = new MetaFilter(modelIds); + metaFilter.setStatus(StatusEnum.ONLINE.getCode()); + Map> metricRespMap = metricService.getMetrics(metaFilter) .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); - Map> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds) + Map> dimensionRespsMap = dimensionService.getDimensions(metaFilter) .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); Map> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds) .stream().collect(Collectors.groupingBy(MeasureResp::getModelId)); for (Long modelId : modelIds) { ModelResp modelResp = getModelMap().get(modelId); - if (modelResp == null) { + if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) { continue; } List measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList()); @@ -318,7 +319,7 @@ public class ModelServiceImpl implements ModelService { private List generateMetricSchema(Long modelId, ModelResp modelResp) { List metricSchemaDescList = new ArrayList<>(); - List metricResps = metricService.getMetrics(modelId); + List metricResps = metricService.getMetrics(new MetaFilter(Lists.newArrayList(modelId))); List measureResps = datasourceService.getMeasureListOfModel(modelId); metricResps.stream().forEach(metricResp -> metricSchemaDescList.add(convert(metricResp, metricResps, measureResps, modelResp))); @@ -326,7 +327,7 @@ public class ModelServiceImpl implements ModelService { } private List generateDimSchema(Long modelId) { - List dimDescList = dimensionService.getDimensions(modelId); + List dimDescList = dimensionService.getDimensions(new MetaFilter(Lists.newArrayList(modelId))); return dimDescList.stream().map(this::convert).collect(Collectors.toList()); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java index 18d5ffb0d..b97442a32 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaRelaResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository; import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.DimensionService; @@ -48,8 +49,11 @@ public class ViewInfoServiceImpl { for (DatasourceResp datasourceResp : datasourceResps) { ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp(); Long datasourceId = datasourceResp.getId(); - List metricResps = metricService.getMetrics(modelId, datasourceId); - List dimensionResps = dimensionService.getDimensionsByDatasource(datasourceId); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); + metaFilter.setDatasourceId(datasourceId); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); domainSchemaRelaResp.setDatasource(datasourceResp); domainSchemaRelaResp.setDimensions(dimensionResps); domainSchemaRelaResp.setMetrics(metricResps); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java index c584ddd74..da7529c57 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java @@ -12,15 +12,16 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceRelaResp; import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.semantic.api.model.response.MeasureResp; +import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; import java.util.List; import java.util.Map; import java.util.Set; public interface DatasourceService { - DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception; + Datasource createDatasource(DatasourceReq datasourceReq, User user) throws Exception; - DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception; + Datasource updateDatasource(DatasourceReq datasourceReq, User user) throws Exception; List getDatasourceListNoMeasurePrefix(Long modelId); @@ -32,7 +33,7 @@ public interface DatasourceService { Map getDatasourceMap(); - void deleteDatasource(Long id) throws Exception; + void deleteDatasource(Long id, User user); DatasourceRelaResp createOrUpdateDatasourceRela(DatasourceRelaReq datasourceRelaReq, User user); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java index 4264fed9b..3e0bda9e8 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java @@ -4,39 +4,32 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; + import java.util.List; public interface DimensionService { - List getDimensions(List ids); - - List getDimensions(Long domainId); - - List getDimensions(); + List getDimensions(MetaFilter metaFilter); DimensionResp getDimension(Long id); DimensionResp getDimension(String bizName, Long modelId); - List getDimensionsByModelIds(List modelIds); + void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); void createDimension(DimensionReq dimensionReq, User user) throws Exception; void createDimensionBatch(List dimensionReqs, User user) throws Exception; - List getDimensionsByDatasource(Long datasourceId); - void updateDimension(DimensionReq dimensionReq, User user) throws Exception; PageInfo queryDimension(PageDimensionReq pageDimensionReq); - List getHighSensitiveDimension(Long domainId); - - List getAllHighSensitiveDimension(); - - void deleteDimension(Long id) throws Exception; + void deleteDimension(Long id, User user); List mockAlias(DimensionReq dimensionReq, String mockType, User user); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java index c079414bd..8e346a01a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java @@ -3,41 +3,32 @@ package com.tencent.supersonic.semantic.model.domain; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; import com.tencent.supersonic.semantic.api.model.response.MetricResp; - +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import java.util.List; import java.util.Set; public interface MetricService { - List getMetrics(List ids); - - List getMetrics(Long modelId); - - List getMetrics(); - - List getMetrics(Long modelId, Long datasourceId); - - void creatExprMetric(MetricReq metricReq, User user) throws Exception; + void createMetric(MetricReq metricReq, User user) throws Exception; void createMetricBatch(List metricReqs, User user) throws Exception; - List getMetricsByModelIds(List modelIds); + void updateExprMetric(MetricReq metricReq, User user) throws Exception; + + void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); + + void deleteMetric(Long id, User user) throws Exception; PageInfo queryMetric(PageMetricReq pageMetricReq, User user); + List getMetrics(MetaFilter metaFilter); + MetricResp getMetric(Long modelId, String bizName); - List getHighSensitiveMetric(Long modelId); - - void updateExprMetric(MetricReq metricReq, User user) throws Exception; - - List getAllHighSensitiveMetric(); - - void deleteMetric(Long id) throws Exception; - List mockAlias(MetricReq metricReq, String mockType, User user); Set getMetricTags(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java index f4458d8ba..2032f4b79 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; - import java.util.List; import java.util.Map; @@ -29,7 +28,7 @@ public interface ModelService { void createModel(ModelReq modelReq, User user); - void deleteModel(Long model); + void deleteModel(Long id, User user); Map getModelMap(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java index 3064b6e55..87ad3c5f2 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java @@ -4,7 +4,7 @@ import java.util.Date; public class DatasourceDO { /** - * + * */ private Long id; @@ -33,6 +33,11 @@ public class DatasourceDO { */ private Long databaseId; + /** + * + */ + private Integer status; + /** * 创建时间 */ @@ -58,18 +63,22 @@ public class DatasourceDO { */ private String datasourceDetail; + /** + * 上游依赖标识 + */ private String depends; - /** - * @return id + * + * @return id */ public Long getId() { return id; } /** - * @param id + * + * @param id */ public void setId(Long id) { this.id = id; @@ -77,7 +86,6 @@ public class DatasourceDO { /** * 主题域ID - * * @return model_id 主题域ID */ public Long getModelId() { @@ -86,7 +94,6 @@ public class DatasourceDO { /** * 主题域ID - * * @param modelId 主题域ID */ public void setModelId(Long modelId) { @@ -95,7 +102,6 @@ public class DatasourceDO { /** * 数据源名称 - * * @return name 数据源名称 */ public String getName() { @@ -104,7 +110,6 @@ public class DatasourceDO { /** * 数据源名称 - * * @param name 数据源名称 */ public void setName(String name) { @@ -113,7 +118,6 @@ public class DatasourceDO { /** * 内部名称 - * * @return biz_name 内部名称 */ public String getBizName() { @@ -122,7 +126,6 @@ public class DatasourceDO { /** * 内部名称 - * * @param bizName 内部名称 */ public void setBizName(String bizName) { @@ -131,7 +134,6 @@ public class DatasourceDO { /** * 数据源描述 - * * @return description 数据源描述 */ public String getDescription() { @@ -140,7 +142,6 @@ public class DatasourceDO { /** * 数据源描述 - * * @param description 数据源描述 */ public void setDescription(String description) { @@ -149,7 +150,6 @@ public class DatasourceDO { /** * 数据库实例ID - * * @return database_id 数据库实例ID */ public Long getDatabaseId() { @@ -158,16 +158,30 @@ public class DatasourceDO { /** * 数据库实例ID - * * @param databaseId 数据库实例ID */ public void setDatabaseId(Long databaseId) { this.databaseId = databaseId; } + /** + * + * @return status + */ + public Integer getStatus() { + return status; + } + + /** + * + * @param status + */ + public void setStatus(Integer status) { + this.status = status; + } + /** * 创建时间 - * * @return created_at 创建时间 */ public Date getCreatedAt() { @@ -176,7 +190,6 @@ public class DatasourceDO { /** * 创建时间 - * * @param createdAt 创建时间 */ public void setCreatedAt(Date createdAt) { @@ -185,7 +198,6 @@ public class DatasourceDO { /** * 创建人 - * * @return created_by 创建人 */ public String getCreatedBy() { @@ -194,7 +206,6 @@ public class DatasourceDO { /** * 创建人 - * * @param createdBy 创建人 */ public void setCreatedBy(String createdBy) { @@ -203,7 +214,6 @@ public class DatasourceDO { /** * 更新时间 - * * @return updated_at 更新时间 */ public Date getUpdatedAt() { @@ -212,7 +222,6 @@ public class DatasourceDO { /** * 更新时间 - * * @param updatedAt 更新时间 */ public void setUpdatedAt(Date updatedAt) { @@ -221,7 +230,6 @@ public class DatasourceDO { /** * 更新人 - * * @return updated_by 更新人 */ public String getUpdatedBy() { @@ -230,7 +238,6 @@ public class DatasourceDO { /** * 更新人 - * * @param updatedBy 更新人 */ public void setUpdatedBy(String updatedBy) { @@ -239,7 +246,6 @@ public class DatasourceDO { /** * 数据源配置 - * * @return datasource_detail 数据源配置 */ public String getDatasourceDetail() { @@ -248,18 +254,25 @@ public class DatasourceDO { /** * 数据源配置 - * * @param datasourceDetail 数据源配置 */ public void setDatasourceDetail(String datasourceDetail) { this.datasourceDetail = datasourceDetail == null ? null : datasourceDetail.trim(); } - public void setDepends(String depends) { - this.depends = depends; - } - + /** + * 上游依赖标识 + * @return depends 上游依赖标识 + */ public String getDepends() { return depends; } -} + + /** + * 上游依赖标识 + * @param depends 上游依赖标识 + */ + public void setDepends(String depends) { + this.depends = depends == null ? null : depends.trim(); + } +} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java index 41882e1fb..0fcf4c1b1 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java @@ -31,6 +31,7 @@ public class DatasourceDOExample { protected Integer limitEnd; /** + * * @mbg.generated */ public DatasourceDOExample() { @@ -38,6 +39,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void setOrderByClause(String orderByClause) { @@ -45,6 +47,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public String getOrderByClause() { @@ -52,6 +55,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void setDistinct(boolean distinct) { @@ -59,6 +63,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public boolean isDistinct() { @@ -66,6 +71,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public List getOredCriteria() { @@ -73,6 +79,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void or(Criteria criteria) { @@ -80,6 +87,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public Criteria or() { @@ -89,6 +97,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public Criteria createCriteria() { @@ -100,6 +109,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ protected Criteria createCriteriaInternal() { @@ -108,6 +118,7 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void clear() { @@ -117,13 +128,15 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void setLimitStart(Integer limitStart) { - this.limitStart = limitStart; + this.limitStart=limitStart; } /** + * * @mbg.generated */ public Integer getLimitStart() { @@ -131,13 +144,15 @@ public class DatasourceDOExample { } /** + * * @mbg.generated */ public void setLimitEnd(Integer limitEnd) { - this.limitEnd = limitEnd; + this.limitEnd=limitEnd; } /** + * * @mbg.generated */ public Integer getLimitEnd() { @@ -578,6 +593,66 @@ public class DatasourceDOExample { return (Criteria) this; } + public Criteria andStatusIsNull() { + addCriterion("status is null"); + return (Criteria) this; + } + + public Criteria andStatusIsNotNull() { + addCriterion("status is not null"); + return (Criteria) this; + } + + public Criteria andStatusEqualTo(Integer value) { + addCriterion("status =", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotEqualTo(Integer value) { + addCriterion("status <>", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThan(Integer value) { + addCriterion("status >", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThanOrEqualTo(Integer value) { + addCriterion("status >=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThan(Integer value) { + addCriterion("status <", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThanOrEqualTo(Integer value) { + addCriterion("status <=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusIn(List values) { + addCriterion("status in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotIn(List values) { + addCriterion("status not in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusBetween(Integer value1, Integer value2) { + addCriterion("status between", value1, value2, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotBetween(Integer value1, Integer value2) { + addCriterion("status not between", value1, value2, "status"); + return (Criteria) this; + } + public Criteria andCreatedAtIsNull() { addCriterion("created_at is null"); return (Criteria) this; @@ -869,6 +944,38 @@ public class DatasourceDOExample { private String typeHandler; + public String getCondition() { + return condition; + } + + public Object getValue() { + return value; + } + + public Object getSecondValue() { + return secondValue; + } + + public boolean isNoValue() { + return noValue; + } + + public boolean isSingleValue() { + return singleValue; + } + + public boolean isBetweenValue() { + return betweenValue; + } + + public boolean isListValue() { + return listValue; + } + + public String getTypeHandler() { + return typeHandler; + } + protected Criterion(String condition) { super(); this.condition = condition; @@ -904,37 +1011,5 @@ public class DatasourceDOExample { protected Criterion(String condition, Object value, Object secondValue) { this(condition, value, secondValue, null); } - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } } -} +} \ No newline at end of file diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java index 974011a68..e1787906e 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java @@ -28,6 +28,11 @@ public class ModelDO { */ private String alias; + /** + * + */ + private String description; + /** * */ @@ -78,6 +83,11 @@ public class ModelDO { */ private String drillDownDimensions; + /** + * + */ + private Integer status; + /** * */ @@ -163,6 +173,22 @@ public class ModelDO { this.alias = alias == null ? null : alias.trim(); } + /** + * + * @return description + */ + public String getDescription() { + return description; + } + + /** + * + * @param description + */ + public void setDescription(String description) { + this.description = description == null ? null : description.trim(); + } + /** * * @return viewer @@ -323,6 +349,22 @@ public class ModelDO { this.drillDownDimensions = drillDownDimensions == null ? null : drillDownDimensions.trim(); } + /** + * + * @return status + */ + public Integer getStatus() { + return status; + } + + /** + * + * @param status + */ + public void setStatus(Integer status) { + this.status = status; + } + /** * * @return entity diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java index fea82e508..8204278fa 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java @@ -533,6 +533,76 @@ public class ModelDOExample { return (Criteria) this; } + public Criteria andDescriptionIsNull() { + addCriterion("description is null"); + return (Criteria) this; + } + + public Criteria andDescriptionIsNotNull() { + addCriterion("description is not null"); + return (Criteria) this; + } + + public Criteria andDescriptionEqualTo(String value) { + addCriterion("description =", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionNotEqualTo(String value) { + addCriterion("description <>", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionGreaterThan(String value) { + addCriterion("description >", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionGreaterThanOrEqualTo(String value) { + addCriterion("description >=", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionLessThan(String value) { + addCriterion("description <", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionLessThanOrEqualTo(String value) { + addCriterion("description <=", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionLike(String value) { + addCriterion("description like", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionNotLike(String value) { + addCriterion("description not like", value, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionIn(List values) { + addCriterion("description in", values, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionNotIn(List values) { + addCriterion("description not in", values, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionBetween(String value1, String value2) { + addCriterion("description between", value1, value2, "description"); + return (Criteria) this; + } + + public Criteria andDescriptionNotBetween(String value1, String value2) { + addCriterion("description not between", value1, value2, "description"); + return (Criteria) this; + } + public Criteria andViewerIsNull() { addCriterion("viewer is null"); return (Criteria) this; @@ -1202,6 +1272,66 @@ public class ModelDOExample { addCriterion("drill_down_dimensions not between", value1, value2, "drillDownDimensions"); return (Criteria) this; } + + public Criteria andStatusIsNull() { + addCriterion("status is null"); + return (Criteria) this; + } + + public Criteria andStatusIsNotNull() { + addCriterion("status is not null"); + return (Criteria) this; + } + + public Criteria andStatusEqualTo(Integer value) { + addCriterion("status =", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotEqualTo(Integer value) { + addCriterion("status <>", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThan(Integer value) { + addCriterion("status >", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThanOrEqualTo(Integer value) { + addCriterion("status >=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThan(Integer value) { + addCriterion("status <", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThanOrEqualTo(Integer value) { + addCriterion("status <=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusIn(List values) { + addCriterion("status in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotIn(List values) { + addCriterion("status not in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusBetween(Integer value1, Integer value2) { + addCriterion("status between", value1, value2, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotBetween(Integer value1, Integer value2) { + addCriterion("status not between", value1, value2, "status"); + return (Criteria) this; + } } /** diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/listener/MetaEmbeddingListener.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/listener/MetaEmbeddingListener.java new file mode 100644 index 000000000..07403b7bd --- /dev/null +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/listener/MetaEmbeddingListener.java @@ -0,0 +1,56 @@ +package com.tencent.supersonic.semantic.model.domain.listener; + +import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.common.pojo.DataEvent; +import com.tencent.supersonic.common.pojo.enums.EventType; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.util.embedding.EmbeddingQuery; +import com.tencent.supersonic.common.util.embedding.EmbeddingUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class MetaEmbeddingListener implements ApplicationListener { + + public static final String COLLECTION_NAME = "meta_collection"; + + @Autowired + private EmbeddingUtils embeddingUtils; + + @Override + public void onApplicationEvent(DataEvent event) { + if (CollectionUtils.isEmpty(event.getDataItems())) { + return; + } + List embeddingQueries = event.getDataItems() + .stream().filter(dataItem -> dataItem.getType().equals(TypeEnums.METRIC)).map(dataItem -> { + EmbeddingQuery embeddingQuery = new EmbeddingQuery(); + embeddingQuery.setQueryId(dataItem.getId().toString()); + embeddingQuery.setQuery(dataItem.getName()); + Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class); + embeddingQuery.setMetadata(meta); + embeddingQuery.setQueryEmbedding(null); + return embeddingQuery; + }).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(embeddingQueries)) { + return; + } + embeddingUtils.addCollection(COLLECTION_NAME); + if (event.getEventType().equals(EventType.ADD)) { + embeddingUtils.addQuery(COLLECTION_NAME, embeddingQueries); + } else if (event.getEventType().equals(EventType.DELETE)) { + embeddingUtils.deleteQuery(COLLECTION_NAME, embeddingQueries); + } else if (event.getEventType().equals(EventType.UPDATE)) { + embeddingUtils.deleteQuery(COLLECTION_NAME, embeddingQueries); + embeddingUtils.addQuery(COLLECTION_NAME, embeddingQueries); + } + } + +} diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java index 02d93bd88..a085da2bb 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java @@ -1,10 +1,12 @@ package com.tencent.supersonic.semantic.model.domain.pojo; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; @Data +@NoArgsConstructor public class MetaFilter { private String id; @@ -23,4 +25,11 @@ public class MetaFilter { private String key; + private List ids; + + private Long datasourceId; + + public MetaFilter(List modelIds) { + this.modelIds = modelIds; + } } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Metric.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Metric.java index 9d45e57d1..c5c1c9524 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Metric.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Metric.java @@ -5,8 +5,6 @@ import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; import java.util.List; @Data @@ -30,11 +28,4 @@ public class Metric extends SchemaItem { private RelateDimension relateDimension; - public String getTag() { - if (CollectionUtils.isEmpty(tags)) { - return ""; - } - return StringUtils.join(tags, ","); - } - } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java index f87eaf82d..476131dfd 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java @@ -19,8 +19,6 @@ public interface DatasourceRepository { DatasourceDO getDatasourceById(Long id); - void deleteDatasource(Long id); - void createDatasourceRela(DatasourceRelaDO datasourceRelaDO); void updateDatasourceRela(DatasourceRelaDO datasourceRelaDO); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java index 9bbae5d5f..bf9d2e171 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java @@ -13,22 +13,9 @@ public interface DimensionRepository { void updateDimension(DimensionDO dimensionDO); - List getDimensionListOfDatasource(Long datasourceId); - - List getDimensionListOfmodel(Long domainId); - - List getDimensionListOfmodelIds(List modelIds); - - List getDimensionList(); - - List getDimensionListByIds(List ids); + void batchUpdateStatus(List dimensionDOS); DimensionDO getDimensionById(Long id); - - List getAllDimensionList(); - List getDimension(DimensionFilter dimensionFilter); - - void deleteDimension(Long id); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java index 630a72f8c..81e24f771 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.semantic.model.domain.repository; import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; - import java.util.List; @@ -15,19 +14,9 @@ public interface MetricRepository { void updateMetric(MetricDO metricDO); - List getMetricList(Long domainId); - - List getMetricList(List modelIds); - - List getMetricList(); - - List getMetricListByIds(List ids); + void batchUpdateStatus(List metricDOS); MetricDO getMetricById(Long id); - List getAllMetricList(); - List getMetric(MetricFilter metricFilter); - - void deleteMetric(Long id); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java index 32f2f6ffa..61f4c875a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java @@ -35,8 +35,8 @@ public class DatasourceConverter { public static Datasource convert(DatasourceReq datasourceReq) { Datasource datasource = new Datasource(); DatasourceDetail datasourceDetail = new DatasourceDetail(); - BeanUtils.copyProperties(datasourceReq, datasource); - BeanUtils.copyProperties(datasourceReq, datasourceDetail); + BeanMapper.mapper(datasourceReq, datasource); + BeanMapper.mapper(datasourceReq, datasourceDetail); List measures = datasourceDetail.getMeasures(); for (Measure measure : measures) { if (StringUtils.isBlank(measure.getExpr())) { @@ -71,7 +71,7 @@ public class DatasourceConverter { public static DatasourceDO convert(Datasource datasource, User user) { DatasourceDO datasourceDO = new DatasourceDO(); - BeanUtils.copyProperties(datasource, datasourceDO); + BeanMapper.mapper(datasource, datasourceDO); datasourceDO.setDatasourceDetail(JSONObject.toJSONString(datasource.getDatasourceDetail())); datasourceDO.setUpdatedBy(user.getName()); datasourceDO.setUpdatedAt(new Date()); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java index 45275ffb8..91869c181 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain.utils; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; @@ -23,38 +24,38 @@ import org.springframework.util.CollectionUtils; public class DimensionConverter { - public static Dimension convert(DimensionReq dimensionReq) { - Dimension dimension = new Dimension(); - BeanUtils.copyProperties(dimensionReq, dimension); - return dimension; - } - - public static DimensionDO convert(DimensionDO dimensionDO, Dimension dimension) { - BeanMapper.mapper(dimension, dimensionDO); - dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues())); - if (!CollectionUtils.isEmpty(dimension.getDimValueMaps())) { - dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps())); + public static DimensionDO convert(DimensionDO dimensionDO, DimensionReq dimensionReq) { + BeanMapper.mapper(dimensionReq, dimensionDO); + if (dimensionReq.getDefaultValues() != null) { + dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues())); + } + if (!CollectionUtils.isEmpty(dimensionReq.getDimValueMaps())) { + dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps())); } else { dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>())); } - if (Objects.nonNull(dimension.getDataType())) { - dimensionDO.setDataType(dimension.getDataType().getType()); + if (Objects.nonNull(dimensionReq.getDataType())) { + dimensionDO.setDataType(dimensionReq.getDataType().getType()); } return dimensionDO; } - public static DimensionDO convert2DimensionDO(Dimension dimension) { + public static DimensionDO convert2DimensionDO(DimensionReq dimensionReq) { DimensionDO dimensionDO = new DimensionDO(); - BeanUtils.copyProperties(dimension, dimensionDO); - dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues())); - dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps())); - if (Objects.nonNull(dimension.getDataType())) { - dimensionDO.setDataType(dimension.getDataType().getType()); + BeanMapper.mapper(dimensionReq, dimensionDO); + if (dimensionReq.getDefaultValues() != null) { + dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues())); } + if (dimensionReq.getDimValueMaps() != null) { + dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps())); + } + if (Objects.nonNull(dimensionReq.getDataType())) { + dimensionDO.setDataType(dimensionReq.getDataType().getType()); + } + dimensionDO.setStatus(StatusEnum.ONLINE.getCode()); return dimensionDO; } - public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO, Map fullPathMap, Map datasourceRespMap) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java index 7b51b195f..b98885bbe 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.domain.utils; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DataFormat; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; @@ -24,24 +25,28 @@ import org.springframework.beans.BeanUtils; public class MetricConverter { - public static Metric convert(MetricReq metricReq) { - Metric metric = new Metric(); - BeanUtils.copyProperties(metricReq, metric); - metric.setType(metricReq.getMetricType().name()); - metric.setTypeParams(metricReq.getTypeParams()); - return metric; + public static MetricDO convert2MetricDO(MetricReq metricReq) { + MetricDO metricDO = new MetricDO(); + BeanMapper.mapper(metricReq, metricDO); + metricDO.setType(metricReq.getMetricType().name()); + metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams())); + metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat())); + metricDO.setTags(metricReq.getTag()); + metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension())); + metricDO.setStatus(StatusEnum.ONLINE.getCode()); + return metricDO; } - public static MetricDO convert(MetricDO metricDO, Metric metric) { - BeanMapper.mapper(metric, metricDO); - metricDO.setTypeParams(JSONObject.toJSONString(metric.getTypeParams())); - if (metric.getDataFormat() != null) { - metricDO.setDataFormat(JSONObject.toJSONString(metric.getDataFormat())); + public static MetricDO convert(MetricDO metricDO, MetricReq metricReq) { + BeanMapper.mapper(metricReq, metricDO); + metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams())); + if (metricReq.getDataFormat() != null) { + metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat())); } - if (metric.getRelateDimension() != null) { - metricDO.setRelateDimensions(JSONObject.toJSONString(metric.getRelateDimension())); + if (metricReq.getRelateDimension() != null) { + metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension())); } - metricDO.setTags(metric.getTag()); + metricDO.setTags(metricReq.getTag()); return metricDO; } @@ -51,17 +56,6 @@ public class MetricConverter { return measureYamlTpl; } - public static MetricDO convert2MetricDO(Metric metric) { - MetricDO metricDO = new MetricDO(); - BeanUtils.copyProperties(metric, metricDO); - metricDO.setTypeParams(JSONObject.toJSONString(metric.getTypeParams())); - metricDO.setDataFormat(JSONObject.toJSONString(metric.getDataFormat())); - metricDO.setTags(metric.getTag()); - metricDO.setRelateDimensions(JSONObject.toJSONString(metric.getRelateDimension())); - return metricDO; - } - - public static MetricResp convert2MetricResp(MetricDO metricDO, Map modelMap) { MetricResp metricResp = new MetricResp(); BeanUtils.copyProperties(metricDO, metricResp); @@ -77,12 +71,6 @@ public class MetricConverter { RelateDimension.class)); return metricResp; } - public static Metric convert2Metric(MetricDO metricDO) { - Metric metric = new Metric(); - BeanUtils.copyProperties(metricDO, metric); - metric.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class)); - return metric; - } public static MetricYamlTpl convert2MetricYamlTpl(Metric metric) { MetricYamlTpl metricYamlTpl = new MetricYamlTpl(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java index 8c5c80a6b..1e9267372 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.semantic.model.domain.utils; import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.Entity; @@ -11,35 +11,19 @@ import com.tencent.supersonic.semantic.api.model.request.ModelReq; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; -import com.tencent.supersonic.semantic.model.domain.pojo.Model; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import java.util.Arrays; -import java.util.Date; import java.util.Map; public class ModelConvert { - public static Model convert(ModelReq modelReq) { - Model model = new Model(); - BeanUtils.copyProperties(modelReq, model); - model.setStatus(StatusEnum.ONLINE.getCode()); - return model; - } - - public static ModelDO convert(Model model, User user) { + public static ModelDO convert(ModelReq modelReq) { ModelDO modelDO = new ModelDO(); - BeanUtils.copyProperties(model, modelDO); - modelDO.setCreatedBy(user.getName()); - modelDO.setUpdatedBy(user.getName()); - modelDO.setCreatedAt(new Date()); - modelDO.setUpdatedAt(new Date()); - modelDO.setAdmin(String.join(",", model.getAdmins())); - modelDO.setAdminOrg(String.join(",", model.getAdminOrgs())); - modelDO.setViewer(String.join(",", model.getViewers())); - modelDO.setViewOrg(String.join(",", model.getViewOrgs())); - modelDO.setEntity(JsonUtil.toString(model.getEntity())); - modelDO.setDrillDownDimensions(JsonUtil.toString(model.getDrillDownDimensions())); + BeanMapper.mapper(modelReq, modelDO); + modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); + modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions())); + modelDO.setStatus(StatusEnum.ONLINE.getCode()); return modelDO; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java index a2bcebea3..312dfb150 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java @@ -13,5 +13,7 @@ public interface DimensionDOCustomMapper { void batchUpdate(List dimensionDOS); + void batchUpdateStatus(List dimensionDOS); + List query(DimensionFilter dimensionFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java index ae8e64d73..d593bc03b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java @@ -13,6 +13,8 @@ public interface MetricDOCustomMapper { void batchUpdate(List metricDOS); + void batchUpdateStatus(List metricDOS); + List query(MetricFilter metricFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java index 61c2d4cb6..577bee0d3 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO; import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDOExample; import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceRelaDO; @@ -7,9 +8,7 @@ import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceRelaDOE import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DatasourceDOMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DatasourceRelaDOMapper; - import java.util.List; - import org.springframework.stereotype.Component; @@ -35,26 +34,29 @@ public class DatasourceRepositoryImpl implements DatasourceRepository { @Override public void updateDatasource(DatasourceDO datasourceDO) { - datasourceMapper.updateByPrimaryKeyWithBLOBs(datasourceDO); + datasourceMapper.updateByPrimaryKeySelective(datasourceDO); } @Override public List getDatasourceList() { DatasourceDOExample datasourceExample = new DatasourceDOExample(); + datasourceExample.createCriteria().andStatusNotEqualTo(StatusEnum.DELETED.getCode()); return datasourceMapper.selectByExampleWithBLOBs(datasourceExample); } @Override public List getDatasourceList(Long modelId) { DatasourceDOExample datasourceExample = new DatasourceDOExample(); - datasourceExample.createCriteria().andModelIdEqualTo(modelId); + datasourceExample.createCriteria().andModelIdEqualTo(modelId) + .andStatusNotEqualTo(StatusEnum.DELETED.getCode()); return datasourceMapper.selectByExampleWithBLOBs(datasourceExample); } @Override public List getDatasourceByDatabase(Long databaseId) { DatasourceDOExample datasourceExample = new DatasourceDOExample(); - datasourceExample.createCriteria().andDatabaseIdEqualTo(databaseId); + datasourceExample.createCriteria().andDatabaseIdEqualTo(databaseId) + .andStatusNotEqualTo(StatusEnum.DELETED.getCode()); return datasourceMapper.selectByExampleWithBLOBs(datasourceExample); } @@ -63,11 +65,6 @@ public class DatasourceRepositoryImpl implements DatasourceRepository { return datasourceMapper.selectByPrimaryKey(id); } - @Override - public void deleteDatasource(Long id) { - datasourceMapper.deleteByPrimaryKey(id); - } - @Override public void createDatasourceRela(DatasourceRelaDO datasourceRelaDO) { datasourceRelaDOMapper.insert(datasourceRelaDO); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java index d8ee7e9af..0511a9de0 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java @@ -1,17 +1,13 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; -import com.google.common.collect.Lists; import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDOExample; import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository; import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOCustomMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOMapper; import java.util.List; -import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; - @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -19,14 +15,12 @@ public class DimensionRepositoryImpl implements DimensionRepository { private DimensionDOCustomMapper dimensionDOCustomMapper; - public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper, DimensionDOCustomMapper dimensionDOCustomMapper) { this.dimensionDOMapper = dimensionDOMapper; this.dimensionDOCustomMapper = dimensionDOCustomMapper; } - @Override public void createDimension(DimensionDO dimensionDO) { dimensionDOMapper.insert(dimensionDO); @@ -39,44 +33,12 @@ public class DimensionRepositoryImpl implements DimensionRepository { @Override public void updateDimension(DimensionDO dimensionDO) { - dimensionDOMapper.updateByPrimaryKeyWithBLOBs(dimensionDO); + dimensionDOMapper.updateByPrimaryKeySelective(dimensionDO); } @Override - public List getDimensionListOfDatasource(Long datasourceId) { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria().andDatasourceIdEqualTo(datasourceId); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); - } - - @Override - public List getDimensionListOfmodel(Long modelId) { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria().andModelIdEqualTo(modelId); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); - } - - @Override - public List getDimensionListOfmodelIds(List modelIds) { - if (CollectionUtils.isEmpty(modelIds)) { - return Lists.newArrayList(); - } - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria().andModelIdIn(modelIds); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); - } - - @Override - public List getDimensionList() { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); - } - - @Override - public List getDimensionListByIds(List ids) { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria().andIdIn(ids); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); + public void batchUpdateStatus(List dimensionDOS) { + dimensionDOCustomMapper.batchUpdateStatus(dimensionDOS); } @Override @@ -84,22 +46,9 @@ public class DimensionRepositoryImpl implements DimensionRepository { return dimensionDOMapper.selectByPrimaryKey(id); } - @Override - public List getAllDimensionList() { - DimensionDOExample dimensionDOExample = new DimensionDOExample(); - return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); - } - - @Override public List getDimension(DimensionFilter dimensionFilter) { return dimensionDOCustomMapper.query(dimensionFilter); } - - @Override - public void deleteDimension(Long id) { - dimensionDOMapper.deleteByPrimaryKey(id); - } - } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java index eb122aa07..378008b57 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java @@ -1,14 +1,11 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; -import com.google.common.collect.Lists; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; -import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDOExample; import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository; import com.tencent.supersonic.semantic.model.infrastructure.mapper.MetricDOCustomMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.MetricDOMapper; import java.util.List; -import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -39,37 +36,12 @@ public class MetricRepositoryImpl implements MetricRepository { @Override public void updateMetric(MetricDO metricDO) { - metricDOMapper.updateByPrimaryKeyWithBLOBs(metricDO); + metricDOMapper.updateByPrimaryKeySelective(metricDO); } @Override - public List getMetricList(Long modelId) { - MetricDOExample metricDOExample = new MetricDOExample(); - metricDOExample.createCriteria().andModelIdEqualTo(modelId); - return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); - } - - @Override - public List getMetricList(List modelIds) { - if (CollectionUtils.isEmpty(modelIds)) { - return Lists.newArrayList(); - } - MetricDOExample metricDOExample = new MetricDOExample(); - metricDOExample.createCriteria().andModelIdIn(modelIds); - return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); - } - - @Override - public List getMetricList() { - MetricDOExample metricDOExample = new MetricDOExample(); - return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); - } - - @Override - public List getMetricListByIds(List ids) { - MetricDOExample metricDOExample = new MetricDOExample(); - metricDOExample.createCriteria().andIdIn(ids); - return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); + public void batchUpdateStatus(List metricDOS) { + metricDOCustomMapper.batchUpdateStatus(metricDOS); } @Override @@ -77,20 +49,9 @@ public class MetricRepositoryImpl implements MetricRepository { return metricDOMapper.selectByPrimaryKey(id); } - @Override - public List getAllMetricList() { - return metricDOMapper.selectByExampleWithBLOBs(new MetricDOExample()); - } - @Override public List getMetric(MetricFilter metricFilter) { return metricDOCustomMapper.query(metricFilter); } - @Override - public void deleteMetric(Long id) { - metricDOMapper.deleteByPrimaryKey(id); - } - - } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java index 403d69602..39d6986ed 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.model.infrastructure.repository; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO; import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDOExample; import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository; @@ -24,17 +25,21 @@ public class ModelRepositoryImpl implements ModelRepository { @Override public void updateModel(ModelDO modelDO) { - modelDOMapper.updateByPrimaryKeyWithBLOBs(modelDO); + modelDOMapper.updateByPrimaryKeySelective(modelDO); } @Override public void deleteModel(Long id) { - modelDOMapper.deleteByPrimaryKey(id); + ModelDO modelDO = modelDOMapper.selectByPrimaryKey(id); + modelDO.setStatus(StatusEnum.DELETED.getCode()); + modelDOMapper.updateByPrimaryKey(modelDO); } @Override public List getModelList() { - return modelDOMapper.selectByExampleWithBLOBs(new ModelDOExample()); + ModelDOExample modelDOExample = new ModelDOExample(); + modelDOExample.createCriteria().andStatusNotEqualTo(StatusEnum.DELETED.getCode()); + return modelDOMapper.selectByExampleWithBLOBs(modelDOExample); } @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java index 35428bc14..2d97a68b8 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.model.domain.DatasourceService; import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.tencent.supersonic.semantic.model.domain.pojo.Datasource; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -24,25 +25,22 @@ import org.springframework.web.bind.annotation.RestController; @RequestMapping("/api/semantic/datasource") public class DatasourceController { - private DatasourceService datasourceService; - public DatasourceController(DatasourceService datasourceService) { this.datasourceService = datasourceService; } @PostMapping("/createDatasource") - public DatasourceResp createDatasource(@RequestBody DatasourceReq datasourceReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + public Datasource createDatasource(@RequestBody DatasourceReq datasourceReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); return datasourceService.createDatasource(datasourceReq, user); } - @PostMapping("/updateDatasource") - public DatasourceResp updateDatasource(@RequestBody DatasourceReq datasourceReq, + public Datasource updateDatasource(@RequestBody DatasourceReq datasourceReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); @@ -59,10 +57,12 @@ public class DatasourceController { return datasourceService.getMeasureListOfModel(modelId); } - @DeleteMapping("deleteDatasource/{id}") - public void deleteDatasource(@PathVariable("id") Long id) throws Exception { - datasourceService.deleteDatasource(id); + public boolean deleteDatasource(@PathVariable("id") Long id, + HttpServletRequest request, HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + datasourceService.deleteDatasource(id, user); + return true; } /** @@ -71,8 +71,7 @@ public class DatasourceController { */ @PostMapping("/createOrUpdateDatasourceRela") public DatasourceRelaResp createOrUpdateDatasourceRela(@RequestBody DatasourceRelaReq datasourceRelaReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return datasourceService.createOrUpdateDatasourceRela(datasourceRelaReq, user); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java index 4e3e851a2..1c73dff65 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java @@ -1,10 +1,13 @@ package com.tencent.supersonic.semantic.model.rest; import com.github.pagehelper.PageInfo; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.model.domain.DimensionService; @@ -13,6 +16,8 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -57,6 +62,15 @@ public class DimensionController { return true; } + @PostMapping("/batchUpdateStatus") + public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + dimensionService.batchUpdateStatus(metaBatchReq, user); + return true; + } + @PostMapping("/mockDimensionAlias") public List mockMetricAlias(@RequestBody DimensionReq dimensionReq, HttpServletRequest request, @@ -76,7 +90,9 @@ public class DimensionController { @GetMapping("/getDimensionList/{modelId}") public List getDimension(@PathVariable("modelId") Long modelId) { - return dimensionService.getDimensions(modelId); + DimensionFilter dimensionFilter = new DimensionFilter(); + dimensionFilter.setModelIds(Lists.newArrayList(modelId)); + return dimensionService.getDimensions(dimensionFilter); } @@ -94,15 +110,20 @@ public class DimensionController { @DeleteMapping("deleteDimension/{id}") - public Boolean deleteDimension(@PathVariable("id") Long id) throws Exception { - dimensionService.deleteDimension(id); + public Boolean deleteDimension(@PathVariable("id") Long id, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + dimensionService.deleteDimension(id, user); return true; } @GetMapping("/getAllHighSensitiveDimension") public List getAllHighSensitiveDimension() { - return dimensionService.getAllHighSensitiveDimension(); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); + return dimensionService.getDimensions(metaFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java index 1267f58ad..276169fee 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java @@ -2,9 +2,12 @@ package com.tencent.supersonic.semantic.model.rest; import com.github.pagehelper.PageInfo; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; +import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; import com.tencent.supersonic.semantic.api.model.response.MetricResp; @@ -14,6 +17,8 @@ import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; +import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RequestMapping; @@ -41,7 +46,7 @@ public class MetricController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - metricService.creatExprMetric(metricReq, user); + metricService.createMetric(metricReq, user); return true; } @@ -54,6 +59,15 @@ public class MetricController { return true; } + @PostMapping("/batchUpdateStatus") + public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + metricService.batchUpdateStatus(metaBatchReq, user); + return true; + } + @PostMapping("/mockMetricAlias") public List mockMetricAlias(@RequestBody MetricReq metricReq, @@ -65,7 +79,8 @@ public class MetricController { @GetMapping("/getMetricList/{modelId}") public List getMetricList(@PathVariable("modelId") Long modelId) { - return metricService.getMetrics(modelId); + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId)); + return metricService.getMetrics(metaFilter); } @@ -84,14 +99,19 @@ public class MetricController { @DeleteMapping("deleteMetric/{id}") - public Boolean deleteMetric(@PathVariable("id") Long id) throws Exception { - metricService.deleteMetric(id); + public Boolean deleteMetric(@PathVariable("id") Long id, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + metricService.deleteMetric(id, user); return true; } @GetMapping("/getAllHighSensitiveMetric") public List getAllHighSensitiveMetric() { - return metricService.getAllHighSensitiveMetric(); + MetricFilter metricFilter = new MetricFilter(); + metricFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); + return metricService.getMetrics(metricFilter); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java index 3c3ec3624..a96c142a1 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java @@ -50,8 +50,11 @@ public class ModelController { } @DeleteMapping("/deleteModel/{modelId}") - public Boolean deleteModel(@PathVariable("modelId") Long modelId) { - modelService.deleteModel(modelId); + public Boolean deleteModel(@PathVariable("modelId") Long modelId, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + modelService.deleteModel(modelId, user); return true; } diff --git a/semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml b/semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml index 35c422518..2e58d0b1d 100644 --- a/semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml +++ b/semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml @@ -8,14 +8,15 @@ + - + @@ -47,11 +48,11 @@ - id, model_id, name, biz_name, description, database_id, created_at, created_by, updated_at, - updated_by,depends + id, model_id, name, biz_name, description, database_id, status, created_at, created_by, + updated_at, updated_by - datasource_detail + datasource_detail, depends @@ -205,6 +220,9 @@ database_id = #{databaseId,jdbcType=BIGINT}, + + status = #{status,jdbcType=INTEGER}, + created_at = #{createdAt,jdbcType=TIMESTAMP}, @@ -220,6 +238,9 @@ datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR}, + + depends = #{depends,jdbcType=LONGVARCHAR}, + where id = #{id,jdbcType=BIGINT} @@ -230,11 +251,13 @@ biz_name = #{bizName,jdbcType=VARCHAR}, description = #{description,jdbcType=VARCHAR}, database_id = #{databaseId,jdbcType=BIGINT}, + status = #{status,jdbcType=INTEGER}, created_at = #{createdAt,jdbcType=TIMESTAMP}, created_by = #{createdBy,jdbcType=VARCHAR}, updated_at = #{updatedAt,jdbcType=TIMESTAMP}, updated_by = #{updatedBy,jdbcType=VARCHAR}, - datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR} + datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR}, + depends = #{depends,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=BIGINT} @@ -244,6 +267,7 @@ biz_name = #{bizName,jdbcType=VARCHAR}, description = #{description,jdbcType=VARCHAR}, database_id = #{databaseId,jdbcType=BIGINT}, + status = #{status,jdbcType=INTEGER}, created_at = #{createdAt,jdbcType=TIMESTAMP}, created_by = #{createdBy,jdbcType=VARCHAR}, updated_at = #{updatedAt,jdbcType=TIMESTAMP}, diff --git a/semantic/model/src/main/resources/mapper/DimensionDOMapper.xml b/semantic/model/src/main/resources/mapper/DimensionDOMapper.xml index 6e9d99892..c454a8176 100644 --- a/semantic/model/src/main/resources/mapper/DimensionDOMapper.xml +++ b/semantic/model/src/main/resources/mapper/DimensionDOMapper.xml @@ -107,7 +107,7 @@ delete from s2_dimension where id = #{id,jdbcType=BIGINT} - + insert into s2_dimension (id, model_id, datasource_id, name, biz_name, description, status, sensitive_level, type, diff --git a/semantic/model/src/main/resources/mapper/MetricDOMapper.xml b/semantic/model/src/main/resources/mapper/MetricDOMapper.xml index e2e1ffbba..0bce5189d 100644 --- a/semantic/model/src/main/resources/mapper/MetricDOMapper.xml +++ b/semantic/model/src/main/resources/mapper/MetricDOMapper.xml @@ -104,7 +104,7 @@ delete from s2_metric where id = #{id,jdbcType=BIGINT} - + insert into s2_metric (id, model_id, name, biz_name, description, status, sensitive_level, type, created_at, diff --git a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml b/semantic/model/src/main/resources/mapper/ModelDOMapper.xml index f4048bf59..486c51018 100644 --- a/semantic/model/src/main/resources/mapper/ModelDOMapper.xml +++ b/semantic/model/src/main/resources/mapper/ModelDOMapper.xml @@ -7,6 +7,7 @@ + @@ -17,6 +18,7 @@ + @@ -80,8 +82,8 @@ - id, name, biz_name, domain_id, alias, viewer, view_org, admin, admin_org, is_open, - created_by, created_at, updated_by, updated_at, drill_down_dimensions + id, name, biz_name, domain_id, alias, description, viewer, view_org, admin, admin_org, + is_open, created_by, created_at, updated_by, updated_at, drill_down_dimensions, status entity @@ -133,17 +135,19 @@ insert into s2_model (id, name, biz_name, - domain_id, alias, viewer, - view_org, admin, admin_org, - is_open, created_by, created_at, - updated_by, updated_at, drill_down_dimensions, - entity) + domain_id, alias, description, + viewer, view_org, admin, + admin_org, is_open, created_by, + created_at, updated_by, updated_at, + drill_down_dimensions, status, entity + ) values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR}, - #{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{viewer,jdbcType=VARCHAR}, - #{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR}, #{adminOrg,jdbcType=VARCHAR}, - #{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR}, #{createdAt,jdbcType=TIMESTAMP}, - #{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, #{drillDownDimensions,jdbcType=VARCHAR}, - #{entity,jdbcType=LONGVARCHAR}) + #{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR}, + #{viewer,jdbcType=VARCHAR}, #{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR}, + #{adminOrg,jdbcType=VARCHAR}, #{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR}, + #{createdAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, + #{drillDownDimensions,jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, #{entity,jdbcType=LONGVARCHAR} + ) insert into s2_model @@ -163,6 +167,9 @@ alias, + + description, + viewer, @@ -193,6 +200,9 @@ drill_down_dimensions, + + status, + entity, @@ -213,6 +223,9 @@ #{alias,jdbcType=VARCHAR}, + + #{description,jdbcType=VARCHAR}, + #{viewer,jdbcType=VARCHAR}, @@ -243,6 +256,9 @@ #{drillDownDimensions,jdbcType=VARCHAR}, + + #{status,jdbcType=INTEGER}, + #{entity,jdbcType=LONGVARCHAR}, @@ -272,6 +288,9 @@ alias = #{record.alias,jdbcType=VARCHAR}, + + description = #{record.description,jdbcType=VARCHAR}, + viewer = #{record.viewer,jdbcType=VARCHAR}, @@ -302,6 +321,9 @@ drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, + + status = #{record.status,jdbcType=INTEGER}, + entity = #{record.entity,jdbcType=LONGVARCHAR}, @@ -317,6 +339,7 @@ biz_name = #{record.bizName,jdbcType=VARCHAR}, domain_id = #{record.domainId,jdbcType=BIGINT}, alias = #{record.alias,jdbcType=VARCHAR}, + description = #{record.description,jdbcType=VARCHAR}, viewer = #{record.viewer,jdbcType=VARCHAR}, view_org = #{record.viewOrg,jdbcType=VARCHAR}, admin = #{record.admin,jdbcType=VARCHAR}, @@ -327,6 +350,7 @@ updated_by = #{record.updatedBy,jdbcType=VARCHAR}, updated_at = #{record.updatedAt,jdbcType=TIMESTAMP}, drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, + status = #{record.status,jdbcType=INTEGER}, entity = #{record.entity,jdbcType=LONGVARCHAR} @@ -339,6 +363,7 @@ biz_name = #{record.bizName,jdbcType=VARCHAR}, domain_id = #{record.domainId,jdbcType=BIGINT}, alias = #{record.alias,jdbcType=VARCHAR}, + description = #{record.description,jdbcType=VARCHAR}, viewer = #{record.viewer,jdbcType=VARCHAR}, view_org = #{record.viewOrg,jdbcType=VARCHAR}, admin = #{record.admin,jdbcType=VARCHAR}, @@ -348,7 +373,8 @@ created_at = #{record.createdAt,jdbcType=TIMESTAMP}, updated_by = #{record.updatedBy,jdbcType=VARCHAR}, updated_at = #{record.updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR} + drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}, + status = #{record.status,jdbcType=INTEGER} @@ -368,6 +394,9 @@ alias = #{alias,jdbcType=VARCHAR}, + + description = #{description,jdbcType=VARCHAR}, + viewer = #{viewer,jdbcType=VARCHAR}, @@ -398,6 +427,9 @@ drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, + + status = #{status,jdbcType=INTEGER}, + entity = #{entity,jdbcType=LONGVARCHAR}, @@ -410,6 +442,7 @@ biz_name = #{bizName,jdbcType=VARCHAR}, domain_id = #{domainId,jdbcType=BIGINT}, alias = #{alias,jdbcType=VARCHAR}, + description = #{description,jdbcType=VARCHAR}, viewer = #{viewer,jdbcType=VARCHAR}, view_org = #{viewOrg,jdbcType=VARCHAR}, admin = #{admin,jdbcType=VARCHAR}, @@ -420,6 +453,7 @@ updated_by = #{updatedBy,jdbcType=VARCHAR}, updated_at = #{updatedAt,jdbcType=TIMESTAMP}, drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, + status = #{status,jdbcType=INTEGER}, entity = #{entity,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=BIGINT} @@ -429,6 +463,7 @@ biz_name = #{bizName,jdbcType=VARCHAR}, domain_id = #{domainId,jdbcType=BIGINT}, alias = #{alias,jdbcType=VARCHAR}, + description = #{description,jdbcType=VARCHAR}, viewer = #{viewer,jdbcType=VARCHAR}, view_org = #{viewOrg,jdbcType=VARCHAR}, admin = #{admin,jdbcType=VARCHAR}, @@ -438,7 +473,8 @@ created_at = #{createdAt,jdbcType=TIMESTAMP}, updated_by = #{updatedBy,jdbcType=VARCHAR}, updated_at = #{updatedAt,jdbcType=TIMESTAMP}, - drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR} + drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}, + status = #{status,jdbcType=INTEGER} where id = #{id,jdbcType=BIGINT} \ No newline at end of file diff --git a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index b2d4358e5..fd5e1bf82 100644 --- a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -64,7 +64,7 @@ created_at, created_by, updated_by, updated_at, semantic_type - + insert into s2_dimension (name, biz_name, description, status, model_id, type, type_params, expr, @@ -107,10 +107,21 @@ + + + update s2_dimension + set + status = #{dimension.status,jdbcType=INTEGER}, + updated_by = #{dimension.updatedBy,jdbcType=VARCHAR}, + updated_at = #{dimension.updatedAt,jdbcType=TIMESTAMP} + where id = #{dimension.id,jdbcType=BIGINT} + + + diff --git a/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml index 19e00624a..466ba6ab3 100644 --- a/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml @@ -62,7 +62,7 @@ type_params - + insert into s2_metric (model_id, name, biz_name, description, type,status,sensitive_level, created_at, created_by, updated_at, @@ -100,10 +100,20 @@ + + + update s2_metric + set status = #{metric.status,jdbcType=INTEGER}, + updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP}, + updated_by = #{metric.updatedBy,jdbcType=VARCHAR} + where id = #{metric.id,jdbcType=BIGINT} + + +