From 6bd8970849bd93622ddcc254a1243673c80f66ba Mon Sep 17 00:00:00 2001 From: WDEP <128273529+WDEP-good@users.noreply.github.com> Date: Sun, 27 Jul 2025 09:02:53 +0800 Subject: [PATCH] Fix model field update (#2343) --- .../chat/knowledge/file/FileHandlerImpl.java | 6 +- .../chat/mapper/EmbeddingMatchStrategy.java | 27 ++- .../mapper/DimensionDOCustomMapper.java | 2 + .../mapper/MetricDOCustomMapper.java | 2 + .../repository/DimensionRepository.java | 2 + .../repository/MetricRepository.java | 2 + .../impl/DimensionRepositoryImpl.java | 5 + .../repository/impl/MetricRepositoryImpl.java | 5 + .../server/service/DimensionService.java | 6 + .../server/service/MetricService.java | 6 + .../headless/server/service/ModelService.java | 6 +- .../service/impl/DimensionServiceImpl.java | 102 ++++++++--- .../service/impl/MetricServiceImpl.java | 164 ++++++++++++------ .../server/service/impl/ModelServiceImpl.java | 153 +++++++++------- .../server/utils/DimensionConverter.java | 3 + .../server/utils/MetricCheckUtils.java | 86 ++++----- .../headless/server/utils/ModelConverter.java | 56 ++---- .../mapper/custom/DimensionDOCustomMapper.xml | 96 +++++----- .../mapper/custom/MetricDOCustomMapper.xml | 43 ++++- .../components/ClassDimensionTable.tsx | 22 +-- .../components/MetricInfoCreateForm.tsx | 9 + .../components/MetricMeasuresFormTable.tsx | 12 +- 22 files changed, 518 insertions(+), 297 deletions(-) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java index 81c877785..2817dea5d 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java @@ -86,7 +86,7 @@ public class FileHandlerImpl implements FileHandler { } private PageInfo getDictValueRespPagWithKey(String fileName, - DictValueReq dictValueReq) { + DictValueReq dictValueReq) { PageInfo dictValueRespPageInfo = new PageInfo<>(); dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); @@ -95,7 +95,7 @@ public class FileHandlerImpl implements FileHandler { Integer startLine = 1; List dictValueRespList = getFileData(filePath, startLine, fileLineNum.intValue()).stream().filter( - dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue())) + dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue())) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dictValueRespList)) { dictValueRespPageInfo.setList(new ArrayList<>()); @@ -118,7 +118,7 @@ public class FileHandlerImpl implements FileHandler { } private PageInfo getDictValueRespPagWithoutKey(String fileName, - DictValueReq dictValueReq) { + DictValueReq dictValueReq) { PageInfo dictValueRespPageInfo = new PageInfo<>(); String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; Long fileLineNum = getFileLineNum(filePath); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java index a7016c558..8fe69766b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java @@ -63,7 +63,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy @Override public List detect(ChatQueryContext chatQueryContext, List terms, - Set detectDataSetIds) { + Set detectDataSetIds) { if (chatQueryContext == null || CollectionUtils.isEmpty(detectDataSetIds)) { log.warn("Invalid input parameters: context={}, dataSetIds={}", chatQueryContext, detectDataSetIds); @@ -92,7 +92,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * Perform enhanced detection using LLM */ private List detectWithLLM(ChatQueryContext chatQueryContext, - Set detectDataSetIds) { + Set detectDataSetIds) { try { String queryText = chatQueryContext.getRequest().getQueryText(); if (StringUtils.isBlank(queryText)) { @@ -126,7 +126,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy @Override public List detectByBatch(ChatQueryContext chatQueryContext, - Set detectDataSetIds, Set detectSegments) { + Set detectDataSetIds, Set detectSegments) { return detectByBatch(chatQueryContext, detectDataSetIds, detectSegments, false); } @@ -140,7 +140,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * @return List of embedding results */ public List detectByBatch(ChatQueryContext chatQueryContext, - Set detectDataSetIds, Set detectSegments, boolean useLlm) { + Set detectDataSetIds, Set detectSegments, boolean useLlm) { Set results = ConcurrentHashMap.newKeySet(); int embeddingMapperBatch = Integer .valueOf(mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH)); @@ -168,7 +168,6 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy variable.put("retrievedInfo", JSONObject.toJSONString(results)); Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable); - ChatModelConfig chatModelConfig=null; if(chatQueryContext.getRequest().getChatAppConfig()!=null && chatQueryContext.getRequest().getChatAppConfig().containsKey("REWRITE_MULTI_TURN")){ @@ -201,7 +200,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * @return Callable task */ private Callable createTask(ChatQueryContext chatQueryContext, Set detectDataSetIds, - List queryTextsSub, Set results, boolean useLlm) { + List queryTextsSub, Set results, boolean useLlm) { return () -> { List oneRoundResults = detectByQueryTextsSub(detectDataSetIds, queryTextsSub, chatQueryContext, useLlm); @@ -222,7 +221,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * @return List of embedding results for this batch */ private List detectByQueryTextsSub(Set detectDataSetIds, - List queryTextsSub, ChatQueryContext chatQueryContext, boolean useLlm) { + List queryTextsSub, ChatQueryContext chatQueryContext, boolean useLlm) { Map> modelIdToDataSetIds = chatQueryContext.getModelIdToDataSetIds(); // Get configuration parameters @@ -244,12 +243,12 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy // Process results List collect = retrieveQueryResults.stream().peek(result -> { - if (!useLlm && CollectionUtils.isNotEmpty(result.getRetrieval())) { - result.getRetrieval() - .removeIf(retrieval -> !result.getQuery().contains(retrieval.getQuery()) - && retrieval.getSimilarity() < threshold); - } - }).filter(result -> CollectionUtils.isNotEmpty(result.getRetrieval())) + if (!useLlm && CollectionUtils.isNotEmpty(result.getRetrieval())) { + result.getRetrieval() + .removeIf(retrieval -> !result.getQuery().contains(retrieval.getQuery()) + && retrieval.getSimilarity() < threshold); + } + }).filter(result -> CollectionUtils.isNotEmpty(result.getRetrieval())) .flatMap(result -> result.getRetrieval().stream() .map(retrieval -> convertToEmbeddingResult(result, retrieval))) .collect(Collectors.toList()); @@ -268,7 +267,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * @return Converted EmbeddingResult */ private EmbeddingResult convertToEmbeddingResult(RetrieveQueryResult queryResult, - Retrieval retrieval) { + Retrieval retrieval) { EmbeddingResult embeddingResult = new EmbeddingResult(); BeanUtils.copyProperties(retrieval, embeddingResult); embeddingResult.setDetectWord(queryResult.getQuery()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index 741220eec..a51b8ef0d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -13,5 +13,7 @@ public interface DimensionDOCustomMapper { void batchUpdateStatus(List dimensionDOS); + void batchUpdate(List dimensionDOS); + List queryDimensions(DimensionsFilter dimensionsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index f53795790..75cd446c8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -13,6 +13,8 @@ public interface MetricDOCustomMapper { void batchUpdateStatus(List metricDOS); + void batchUpdate(List metricDOS); + void batchPublish(List metricDOS); void batchUnPublish(List metricDOS); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java index 82153d129..af64404bb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java @@ -16,6 +16,8 @@ public interface DimensionRepository { void batchUpdateStatus(List dimensionDOS); + void batchUpdate(List dimensionDOS); + DimensionDO getDimensionById(Long id); List getDimension(DimensionFilter dimensionFilter); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java index 69a750282..ac58e9bc0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java @@ -17,6 +17,8 @@ public interface MetricRepository { void batchUpdateStatus(List metricDOS); + void batchUpdateMetric(List metricDOS); + void batchPublish(List metricDOS); void batchUnPublish(List metricDOS); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index 7ba0d3ebf..512a3e977 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -46,6 +46,11 @@ public class DimensionRepositoryImpl implements DimensionRepository { dimensionDOCustomMapper.batchUpdateStatus(dimensionDOS); } + @Override + public void batchUpdate(List dimensionDOS) { + dimensionDOCustomMapper.batchUpdate(dimensionDOS); + } + @Override public DimensionDO getDimensionById(Long id) { return dimensionDOMapper.selectById(id); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index df4e11e70..181bf0f07 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -53,6 +53,11 @@ public class MetricRepositoryImpl implements MetricRepository { metricDOCustomMapper.batchUpdateStatus(metricDOS); } + @Override + public void batchUpdateMetric(List metricDOS) { + metricDOCustomMapper.batchUpdate(metricDOS); + } + @Override public void batchPublish(List metricDOS) { metricDOCustomMapper.batchPublish(metricDOS); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java index 864757002..d954f225a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java @@ -27,10 +27,14 @@ public interface DimensionService { DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception; + void alterDimensionBatch(List dimensionReqs, Long modelId, User user) throws Exception; + void createDimensionBatch(List dimensionReqs, User user) throws Exception; void updateDimension(DimensionReq dimensionReq, User user) throws Exception; + void updateDimensionBatch(List dimensionReqs, User user) throws Exception; + PageInfo queryDimension(PageDimensionReq pageDimensionReq); List queryDimensions(DimensionsFilter dimensionsFilter); @@ -39,6 +43,8 @@ public interface DimensionService { void deleteDimension(Long id, User user); + void deleteDimensionBatch(List idList, User user); + List getDimensionInModelCluster(Long modelId); List mockAlias(DimensionReq dimensionReq, String mockType, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java index 49e254dd3..c45c9a460 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java @@ -26,8 +26,12 @@ public interface MetricService { void createMetricBatch(List metricReqs, User user) throws Exception; + void alterMetricBatch(List metricReqs, Long modelId, User user) throws Exception; + MetricResp updateMetric(MetricReq metricReq, User user) throws Exception; + void updateMetricBatch(List metricReqs, User user) throws Exception; + void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); void batchPublish(List metricIds, User user); @@ -40,6 +44,8 @@ public interface MetricService { void deleteMetric(Long id, User user) throws Exception; + void deleteMetricBatch(List idList, User user); + PageInfo queryMetricMarket(PageMetricReq pageMetricReq, User user); PageInfo queryMetric(PageMetricReq pageMetricReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index 6bf019387..dbfdc3920 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -11,6 +11,8 @@ import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import java.sql.SQLException; @@ -53,5 +55,7 @@ public interface ModelService { void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); - Dimension updateDimension(DimensionReq dimensionReq, User user); + void updateModelByDimAndMetric(Long modelId, List dimensionReqList, List metricReqList, User user); + + void deleteModelDetailByDimAndMetric(Long modelId, List dimensionReqList, List metricReqList); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 0bd154ccb..9040a8b4b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -67,8 +67,8 @@ public class DimensionServiceImpl extends ServiceImpl dimensionReqs, User user) { - if (CollectionUtils.isEmpty(dimensionReqs)) { - return; - } - Long modelId = dimensionReqs.get(0).getModelId(); + public void alterDimensionBatch(List dimensionReqs, Long modelId, User user) throws Exception { List dimensionResps = getDimensions(modelId); + // get all dimension in model, only use bizname, because name can be changed to everything Map bizNameMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1)); List dimensionToInsert = Lists.newArrayList(); + List dimensionToUpdate = Lists.newArrayList(); + List dimensionToDelete = Lists.newArrayList(); + + // look for which dimension need to insert, update, delete dimensionReqs.stream().forEach(dimension -> { - if (!bizNameMap.containsKey(dimension.getBizName()) - && !nameMap.containsKey(dimension.getName())) { + if (!bizNameMap.containsKey(dimension.getBizName())) { dimensionToInsert.add(dimension); } else { DimensionResp dimensionRespByBizName = bizNameMap.get(dimension.getBizName()); - DimensionResp dimensionRespByName = nameMap.get(dimension.getName()); if (null != dimensionRespByBizName && isChange(dimension, dimensionRespByBizName)) { dimension.setId(dimensionRespByBizName.getId()); - this.updateDimension(dimension, user); - } else { - if (null != dimensionRespByName && isChange(dimension, dimensionRespByName)) { - dimension.setId(dimensionRespByName.getId()); - this.updateDimension(dimension, user); - } + dimension.updatedBy(user.getName()); + dimensionToUpdate.add(dimension); } } }); - if (CollectionUtils.isEmpty(dimensionToInsert)) { - return; + + // the bizNames from alter dimensions + List bizNames = dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList()); + bizNameMap.keySet().forEach(bizNameInDb -> { + if (!bizNames.contains(bizNameInDb)) { + dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId()); + } + }); + + // insert + if (!CollectionUtils.isEmpty(dimensionToInsert)) { + createDimensionBatch(dimensionToInsert, user); } + + // update + if (!CollectionUtils.isEmpty(dimensionToUpdate)) { + updateDimensionBatch(dimensionToUpdate, user); + } + + // delete + if (!CollectionUtils.isEmpty(dimensionToDelete)) { + deleteDimensionBatch(dimensionToDelete, user); + } + + } + + @Override + public void createDimensionBatch(List dimensionReqs, User user) { List dimensionDOS = - dimensionToInsert.stream().peek(dimension -> dimension.createdBy(user.getName())) + dimensionReqs.stream().peek(dimension -> dimension.createdBy(user.getName())) .map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList()); dimensionRepository.createDimensionBatch(dimensionDOS); + // should update modelDetail as well + modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs, null, user); + sendEventBatch(dimensionDOS, EventType.ADD); } @@ -141,13 +162,23 @@ public class DimensionServiceImpl extends ServiceImpl dimensionReqList, User user) { + checkExist(dimensionReqList); + List dimensionDOS = dimensionReqList.stream().map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList()); + dimensionRepository.batchUpdate(dimensionDOS); + // should update modelDetail as well + modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),dimensionReqList, null, user); + sendEventBatch(dimensionDOS, EventType.UPDATE); + } + @Override public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) { if (CollectionUtils.isEmpty(metaBatchReq.getIds())) { @@ -199,9 +230,30 @@ public class DimensionServiceImpl extends ServiceImpl idList, User user) { + DimensionsFilter dimensionFilter = new DimensionsFilter(); + dimensionFilter.setDimensionIds(idList); + List dimensionDOList = dimensionRepository.getDimensions(dimensionFilter); + if (CollectionUtils.isEmpty(dimensionDOList)) { + throw new RuntimeException(String.format("the dimension %s not exist", StringUtils.join(",",idList))); + } + dimensionDOList.forEach(dimensionDO -> { + dimensionDO.setStatus(StatusEnum.DELETED.getCode()); + dimensionDO.setUpdatedAt(new Date()); + dimensionDO.setUpdatedBy(user.getName()); + }); + dimensionRepository.batchUpdateStatus(dimensionDOList); + // should update modelDetail + modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(), dimensionDOList, null); + sendEventBatch(dimensionDOList, EventType.DELETE); + } + @Override public DimensionResp getDimension(String bizName, Long modelId) { List dimensionResps = getDimensions(modelId); @@ -271,7 +323,7 @@ public class DimensionServiceImpl extends ServiceImpl filterByField(List dimensionResps, - List fields) { + List fields) { List dimensionFiltered = Lists.newArrayList(); for (DimensionResp dimensionResp : dimensionResps) { for (String field : fields) { @@ -306,7 +358,7 @@ public class DimensionServiceImpl extends ServiceImpl dimensionResps = Lists.newArrayList(); if (!CollectionUtils.isEmpty(dimensionDOS)) { dimensionResps = dimensionDOS.stream().map( - dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap)) + dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap)) .collect(Collectors.toList()); } return dimensionResps; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index a754380fc..2d130833e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -11,23 +11,20 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; -import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO; -import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; -import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.*; import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.*; import com.tencent.supersonic.headless.server.service.*; -import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; -import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; -import com.tencent.supersonic.headless.server.utils.MetricConverter; -import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; +import com.tencent.supersonic.headless.server.utils.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -61,9 +58,9 @@ public class MetricServiceImpl extends ServiceImpl private ChatLayerService chatLayerService; public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, - AliasGenerateHelper aliasGenerateHelper, CollectService collectService, - DataSetService dataSetService, ApplicationEventPublisher eventPublisher, - DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) { + AliasGenerateHelper aliasGenerateHelper, CollectService collectService, + DataSetService dataSetService, ApplicationEventPublisher eventPublisher, + DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) { this.metricRepository = metricRepository; this.modelService = modelService; this.aliasGenerateHelper = aliasGenerateHelper; @@ -77,57 +74,82 @@ public class MetricServiceImpl extends ServiceImpl @Override public MetricResp createMetric(MetricReq metricReq, User user) { checkExist(Lists.newArrayList(metricReq)); - MetricCheckUtils.checkParam(metricReq); + MetricCheckUtils.checkParam(Lists.newArrayList(metricReq)); metricReq.createdBy(user.getName()); MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq); metricRepository.createMetric(metricDO); sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD); + // should update modelDetail as well + modelService.updateModelByDimAndMetric(metricReq.getModelId(),null, Lists.newArrayList(metricReq), user); + + return MetricConverter.convert2MetricResp(metricDO); } @Override public void createMetricBatch(List metricReqs, User user) { - if (CollectionUtils.isEmpty(metricReqs)) { - return; - } - Long modelId = metricReqs.get(0).getModelId(); - List metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId))); - Map bizNameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); - Map nameMap = metricResps.stream() - .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); - List metricToInsert = Lists.newArrayList(); - metricReqs.stream().forEach(metric -> { - if (!bizNameMap.containsKey(metric.getBizName()) - && !nameMap.containsKey(metric.getName())) { - metricToInsert.add(metric); - } else { - MetricResp metricRespByBizName = bizNameMap.get(metric.getBizName()); - MetricResp metricRespByName = nameMap.get(metric.getName()); - if (null != metricRespByBizName) { - metric.setId(metricRespByBizName.getId()); - this.updateMetric(metric, user); - } else { - if (null != metricRespByName) { - metric.setId(metricRespByName.getId()); - this.updateMetric(metric, user); - } - } - } - }); - if (CollectionUtils.isEmpty(metricToInsert)) { - return; - } List metricDOS = - metricToInsert.stream().peek(metric -> metric.createdBy(user.getName())) + metricReqs.stream().peek(metric -> metric.createdBy(user.getName())) .map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); metricRepository.createMetricBatch(metricDOS); + // should update modelDetail as well + modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user); + sendEventBatch(metricDOS, EventType.ADD); } + @Override + public void alterMetricBatch(List metricReqs, Long modelId, User user) { + List metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId))); + // get all metric in model, only use bizname, because name can be changed to everything + Map bizNameMap = metricResps.stream() + .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); + + List metricToInsert = Lists.newArrayList(); + List metricToUpdate = Lists.newArrayList(); + List metricToDelete = Lists.newArrayList(); + + metricReqs.stream().forEach(metric -> { + if (!bizNameMap.containsKey(metric.getBizName())) { + metricToInsert.add(metric); + } else { + MetricResp metricRespByBizName = bizNameMap.get(metric.getBizName()); + if (null != metricRespByBizName && isChange(metric, metricRespByBizName)) { + metric.setId(metricRespByBizName.getId()); + metric.updatedBy(user.getName()); + metricToUpdate.add(metric); + } + } + }); + + // the bizNames from alter dimensions + List bizNames = metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList()); + bizNameMap.keySet().forEach(bizNameInDb -> { + if (!bizNames.contains(bizNameInDb)) { + metricToDelete.add(bizNameMap.get(bizNameInDb).getId()); + } + }); + + // insert + if (!CollectionUtils.isEmpty(metricToInsert)) { + createMetricBatch(metricToInsert, user); + } + + // update + if (!CollectionUtils.isEmpty(metricToUpdate)) { + updateMetricBatch(metricToUpdate, user); + } + + // delete + if (!CollectionUtils.isEmpty(metricToDelete)) { + deleteMetricBatch(metricToDelete, user); + } + + } + @Override public MetricResp updateMetric(MetricReq metricReq, User user) { - MetricCheckUtils.checkParam(metricReq); + MetricCheckUtils.checkParam(Lists.newArrayList(metricReq)); checkExist(Lists.newArrayList(metricReq)); metricReq.updatedBy(user.getName()); MetricDO metricDO = metricRepository.getMetricById(metricReq.getId()); @@ -140,9 +162,23 @@ public class MetricServiceImpl extends ServiceImpl dataItem.setNewName(metricDO.getName()); sendEvent(dataItem, EventType.UPDATE); } + // should update modelDetail as well + modelService.updateModelByDimAndMetric(metricReq.getModelId(), null, Lists.newArrayList(metricReq), user); return MetricConverter.convert2MetricResp(metricDO); } + @Override + public void updateMetricBatch(List metricReqs, User user) { + MetricCheckUtils.checkParam(metricReqs); + checkExist(metricReqs); + List metricDOS = metricReqs.stream().map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); + metricRepository.batchUpdateMetric(metricDOS); + // should update modelDetail as well + modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user); + sendEventBatch(metricDOS, EventType.UPDATE); + } + + @Override public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) { if (CollectionUtils.isEmpty(metaBatchReq.getIds())) { @@ -240,9 +276,30 @@ public class MetricServiceImpl extends ServiceImpl metricDO.setUpdatedAt(new Date()); metricDO.setUpdatedBy(user.getName()); metricRepository.updateMetric(metricDO); + // should update modelDetail + modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null, Lists.newArrayList(metricDO)); sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE); } + @Override + public void deleteMetricBatch(List idList, User user) { + MetricsFilter metricsFilter = new MetricsFilter(); + metricsFilter.setMetricIds(idList); + List metricDOList = metricRepository.getMetrics(metricsFilter); + if (CollectionUtils.isEmpty(metricDOList)) { + throw new RuntimeException(String.format("the metrics %s not exist", StringUtils.join(",",idList))); + } + metricDOList.forEach(metricDO -> { + metricDO.setStatus(StatusEnum.DELETED.getCode()); + metricDO.setUpdatedAt(new Date()); + metricDO.setUpdatedBy(user.getName()); + }); + metricRepository.batchUpdateStatus(metricDOList); + // should update modelDetail + modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null, metricDOList); + sendEventBatch(metricDOList, EventType.DELETE); + } + @Override public PageInfo queryMetricMarket(PageMetricReq pageMetricReq, User user) { // search by whole text @@ -376,7 +433,7 @@ public class MetricServiceImpl extends ServiceImpl } private boolean filterByField(List metricResps, MetricResp metricResp, - List fields, Set metricRespFiltered) { + List fields, Set metricRespFiltered) { if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { List ids = metricResp.getMetricDefineByMetricParams().getMetrics().stream() .map(MetricParam::getId).collect(Collectors.toList()); @@ -413,8 +470,8 @@ public class MetricServiceImpl extends ServiceImpl metricFilter.setModelIds(Lists.newArrayList(modelId)); List metricResps = getMetrics(metricFilter); return metricResps.stream().filter( - metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType()) - || MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) + metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType()) + || MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) .collect(Collectors.toList()); } @@ -549,7 +606,7 @@ public class MetricServiceImpl extends ServiceImpl return metricQueryDefaultConfig; } - private void checkExist(List metricReqs) { + private void checkExist(List metricReqs) { Long modelId = metricReqs.get(0).getModelId(); MetaFilter metaFilter = new MetaFilter(); metaFilter.setModelIds(Lists.newArrayList(modelId)); @@ -588,7 +645,7 @@ public class MetricServiceImpl extends ServiceImpl Map modelMap = modelService.getModelMap(modelFilter); if (!CollectionUtils.isEmpty(metricDOS)) { metricResps = metricDOS.stream().map( - metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect)) + metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect)) .collect(Collectors.toList()); } return metricResps; @@ -646,7 +703,7 @@ public class MetricServiceImpl extends ServiceImpl @Override public void batchFillMetricDefaultAgg(List metricResps, - List modelResps) { + List modelResps) { Map modelRespMap = modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m)); for (MetricResp metricResp : metricResps) { @@ -795,7 +852,7 @@ public class MetricServiceImpl extends ServiceImpl } private Set getModelIds(Set modelIdsByDomainId, List metricResps, - List dimensionResps) { + List dimensionResps) { Set result = new HashSet<>(); if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) { result.addAll(modelIdsByDomainId); @@ -830,8 +887,9 @@ public class MetricServiceImpl extends ServiceImpl return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); } - private boolean isNameChange(MetricReq metricReq, MetricResp metricResp) { + private boolean isChange(MetricReq metricReq, MetricResp metricResp) { boolean isNameChange = !metricReq.getName().equals(metricResp.getName()); - return isNameChange; + boolean isBizNameChange = !Objects.equals(metricReq.getMetricDefineByMeasureParams(),metricResp.getMetricDefineByMeasureParams()); + return isNameChange || isBizNameChange; } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index aed149841..4c0b61535 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -15,6 +15,8 @@ import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.modeller.SemanticModeller; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; +import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; import com.tencent.supersonic.headless.server.persistence.repository.ModelRepository; @@ -65,10 +67,10 @@ public class ModelServiceImpl implements ModelService { new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService, - @Lazy DimensionService dimensionService, @Lazy MetricService metricService, - DomainService domainService, UserService userService, DataSetService dataSetService, - DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService, - ApplicationEventPublisher eventPublisher) { + @Lazy DimensionService dimensionService, @Lazy MetricService metricService, + DomainService domainService, UserService userService, DataSetService dataSetService, + DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService, + ApplicationEventPublisher eventPublisher) { this.modelRepository = modelRepository; this.databaseService = databaseService; this.dimensionService = dimensionService; @@ -87,8 +89,12 @@ public class ModelServiceImpl implements ModelService { // checkParams(modelReq); ModelDO modelDO = ModelConverter.convert(modelReq, user); modelRepository.createModel(modelDO); - batchCreateDimension(modelDO, user); - batchCreateMetric(modelDO, user); + // create or update dimension + List dimensionReqs = ModelConverter.convertDimensionList(modelDO); + dimensionService.alterDimensionBatch(dimensionReqs, modelDO.getId(), user); + // create or update metric + List metricReqs = ModelConverter.convertMetricList(modelDO); + metricService.alterMetricBatch(metricReqs, modelDO.getId(), user); sendEvent(modelDO, EventType.ADD); return ModelConverter.convert(modelDO); } @@ -117,8 +123,12 @@ public class ModelServiceImpl implements ModelService { ModelDO modelDO = modelRepository.getModelById(modelReq.getId()); ModelConverter.convert(modelDO, modelReq, user); modelRepository.updateModel(modelDO); - batchCreateDimension(modelDO, user); - batchCreateMetric(modelDO, user); + // create or update dimension + List dimensionReqs = ModelConverter.convertDimensionList(modelDO); + dimensionService.alterDimensionBatch(dimensionReqs, modelDO.getId(), user); + // create or update metric + List metricReqs = ModelConverter.convertMetricList(modelDO); + metricService.alterMetricBatch(metricReqs, modelDO.getId(), user); sendEvent(modelDO, EventType.UPDATE); return ModelConverter.convert(modelDO); } @@ -206,7 +216,7 @@ public class ModelServiceImpl implements ModelService { } private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List dbSchemas, - Map modelSchemaMap) { + Map modelSchemaMap) { ModelSchema modelSchema = new ModelSchema(); List semanticModellers = CoreComponentFactory.getSemanticModellers(); for (SemanticModeller semanticModeller : semanticModellers) { @@ -224,7 +234,7 @@ public class ModelServiceImpl implements ModelService { } private List convert(Map> dbColumnMap, - ModelBuildReq modelBuildReq) { + ModelBuildReq modelBuildReq) { return dbColumnMap.keySet().stream() .map(key -> convert(modelBuildReq, key, dbColumnMap.get(key))) .collect(Collectors.toList()); @@ -239,32 +249,6 @@ public class ModelServiceImpl implements ModelService { return dbSchema; } - private void batchCreateDimension(ModelDO modelDO, User user) throws Exception { - List dimensionReqs = ModelConverter.convertDimensionList(modelDO); - List dimensionToCreate = new ArrayList<>(); - for (DimensionReq dimensionReq : dimensionReqs) { - DimensionResp dimensionResp = - dimensionService.getDimension(dimensionReq.getBizName(), modelDO.getId()); - if (dimensionResp == null) { - dimensionToCreate.add(dimensionReq); - } - } - dimensionService.createDimensionBatch(dimensionToCreate, user); - } - - private void batchCreateMetric(ModelDO modelDO, User user) throws Exception { - List metricReqs = ModelConverter.convertMetricList(modelDO); - List metricToCreate = new ArrayList<>(); - for (MetricReq metricReq : metricReqs) { - MetricResp metricResp = - metricService.getMetric(modelDO.getId(), metricReq.getBizName()); - if (metricResp == null) { - metricToCreate.add(metricReq); - } - } - metricService.createMetricBatch(metricToCreate, user); - } - private void checkParams(ModelReq modelReq) { String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(modelReq.getName()); if (StringUtils.isNotBlank(forbiddenCharacters)) { @@ -399,7 +383,7 @@ public class ModelServiceImpl implements ModelService { } public List getModelRespAuthInheritDomain(User user, Long domainId, - AuthType authType) { + AuthType authType) { List domainIds = domainService.getDomainAuthSet(user, authType).stream().filter(domainResp -> { if (domainId == null) { @@ -537,32 +521,83 @@ public class ModelServiceImpl implements ModelService { } @Override - public Dimension updateDimension(DimensionReq dimensionReq, User user) { - ModelDO modelDO = getModelDO(dimensionReq.getModelId()); + public void updateModelByDimAndMetric(Long modelId, List dimensionReqList, List metricReqList, User user) { + ModelDO modelDO = getModelDO(modelId); ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class); - Optional dimOptional = modelDetail.getDimensions().stream() - .filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName())) - .findFirst(); - Dimension result; - if (dimOptional.isPresent()) { - Dimension dimension = dimOptional.get(); - dimension.setExpr(dimensionReq.getExpr()); - dimension.setName(dimensionReq.getName()); - dimension.setType(DimensionType.valueOf(dimensionReq.getType())); - dimension.setDescription(dimensionReq.getDescription()); - result = dimension; - } else { - Dimension dimension = Dimension.builder().name(dimensionReq.getName()) - .bizName(dimensionReq.getBizName()).expr(dimensionReq.getExpr()) - .type(DimensionType.valueOf(dimensionReq.getType())) - .description(dimensionReq.getDescription()).build(); - modelDetail.getDimensions().add(dimension); - result = dimension; + if (!CollectionUtils.isEmpty(dimensionReqList)) { + dimensionReqList.forEach(dimensionReq -> { + Optional dimOptional = modelDetail.getDimensions().stream() + .filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName())) + .findFirst(); + if (dimOptional.isPresent()) { + Dimension dimension = dimOptional.get(); + dimension.setExpr(dimensionReq.getExpr()); + dimension.setName(dimensionReq.getName()); + dimension.setType(DimensionType.valueOf(dimensionReq.getType())); + dimension.setDescription(dimensionReq.getDescription()); + } else { + Dimension dimension = Dimension.builder().name(dimensionReq.getName()) + .bizName(dimensionReq.getBizName()).expr(dimensionReq.getExpr()) + .type(DimensionType.valueOf(dimensionReq.getType())) + .description(dimensionReq.getDescription()).build(); + modelDetail.getDimensions().add(dimension); + } + }); + } + + if (!CollectionUtils.isEmpty(metricReqList)) { + // 目前modeltail中的measure + Map mesureMap = modelDetail.getMeasures().stream().collect(Collectors.toMap(Measure::getBizName, a -> a, (k1, k2) -> k1)); + metricReqList.forEach(metricReq -> { + if (null != metricReq.getMetricDefineByMeasureParams() && !CollectionUtils.isEmpty(metricReq.getMetricDefineByMeasureParams().getMeasures())) { + for(Measure alterMeasure : metricReq.getMetricDefineByMeasureParams().getMeasures()) { + if (mesureMap.containsKey(alterMeasure.getBizName())) { + Measure measure = mesureMap.get(alterMeasure.getBizName()); + BeanUtils.copyProperties(alterMeasure, measure); + } else { + modelDetail.getMeasures().add(alterMeasure); + } + } + } else { + modelDetail.getMeasures().clear(); + } + }); + } + + modelDO.setModelDetail(JsonUtil.toString(modelDetail)); + modelRepository.updateModel(modelDO); + } + + @Override + public void deleteModelDetailByDimAndMetric(Long modelId, List dimensionList, List metricReqList) { + ModelDO modelDO = getModelDO(modelId); + ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class); + if (!CollectionUtils.isEmpty(dimensionList)) { + dimensionList.forEach(dimensionReq -> { + Optional dimOptional = modelDetail.getDimensions().stream() + .filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName())) + .findFirst(); + if (dimOptional.isPresent()) { + Dimension dimension = dimOptional.get(); + modelDetail.getDimensions().remove(dimension); + } + }); + } + + if (!CollectionUtils.isEmpty(metricReqList)) { + metricReqList.forEach(metricReq -> { + Optional metricOptional = modelDetail.getMeasures().stream() + .filter(measure -> measure.getBizName().equals(metricReq.getBizName())) + .findFirst(); + if (metricOptional.isPresent()) { + Measure measure = metricOptional.get(); + modelDetail.getMeasures().remove(measure); + } + }); } modelDO.setModelDetail(JsonUtil.toString(modelDetail)); modelRepository.updateModel(modelDO); - return result; } protected ModelDO getModelDO(Long id) { @@ -603,7 +638,7 @@ public class ModelServiceImpl implements ModelService { } public static boolean checkDataSetPermission(Set orgIds, User user, - ModelResp modelResp) { + ModelResp modelResp) { if (checkAdminPermission(orgIds, user, modelResp)) { return true; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index a5e17be93..d6893211c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -70,6 +70,9 @@ public class DimensionConverter { if (dimensionReq.getExt() != null) { dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt())); } + if (dimensionReq.getTypeParams() != null) { + dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams())); + } dimensionDO.setStatus(StatusEnum.ONLINE.getCode()); return dimensionDO; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java index 55b372be4..9b0d2729c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java @@ -10,54 +10,58 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; +import java.util.List; + public class MetricCheckUtils { - public static void checkParam(MetricReq metricReq) { - String expr = ""; - if (MetricDefineType.METRIC.equals(metricReq.getMetricDefineType())) { - MetricDefineByMetricParams typeParams = metricReq.getMetricDefineByMetricParams(); - if (typeParams == null) { - throw new InvalidArgumentException("指标定义参数不可为空"); + public static void checkParam(List metricReqList) { + metricReqList.forEach(metricReq -> { + String expr = ""; + if (MetricDefineType.METRIC.equals(metricReq.getMetricDefineType())) { + MetricDefineByMetricParams typeParams = metricReq.getMetricDefineByMetricParams(); + if (typeParams == null) { + throw new InvalidArgumentException("指标定义参数不可为空"); + } + expr = typeParams.getExpr(); + if (CollectionUtils.isEmpty(typeParams.getMetrics())) { + throw new InvalidArgumentException("定义指标的指标列表参数不可为空"); + } + if (hasAggregateFunction(expr)) { + throw new InvalidArgumentException("基于指标来创建指标,表达式中不可再包含聚合函数"); + } } - expr = typeParams.getExpr(); - if (CollectionUtils.isEmpty(typeParams.getMetrics())) { - throw new InvalidArgumentException("定义指标的指标列表参数不可为空"); + if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) { + MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams(); + if (typeParams == null) { + throw new InvalidArgumentException("指标定义参数不可为空"); + } + expr = typeParams.getExpr(); + if (hasAggregateFunction(expr)) { + throw new InvalidArgumentException("基于度量来创建指标,表达式中不可再包含聚合函数"); + } } - if (hasAggregateFunction(expr)) { - throw new InvalidArgumentException("基于指标来创建指标,表达式中不可再包含聚合函数"); + if (MetricDefineType.FIELD.equals(metricReq.getMetricDefineType())) { + MetricDefineByFieldParams typeParams = metricReq.getMetricDefineByFieldParams(); + if (typeParams == null) { + throw new InvalidArgumentException("指标定义参数不可为空"); + } + expr = typeParams.getExpr(); + // if (CollectionUtils.isEmpty(typeParams.getFields())) { + // throw new InvalidArgumentException("定义指标的字段列表参数不可为空"); + // } + if (!hasAggregateFunction(expr)) { + throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数"); + } } - } - if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) { - MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams(); - if (typeParams == null) { - throw new InvalidArgumentException("指标定义参数不可为空"); + if (StringUtils.isBlank(expr)) { + throw new InvalidArgumentException("表达式不可为空"); } - expr = typeParams.getExpr(); - if (hasAggregateFunction(expr)) { - throw new InvalidArgumentException("基于度量来创建指标,表达式中不可再包含聚合函数"); + String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(metricReq.getName()); + if (StringUtils.isNotBlank(forbiddenCharacters)) { + throw new InvalidArgumentException( + String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters)); } - } - if (MetricDefineType.FIELD.equals(metricReq.getMetricDefineType())) { - MetricDefineByFieldParams typeParams = metricReq.getMetricDefineByFieldParams(); - if (typeParams == null) { - throw new InvalidArgumentException("指标定义参数不可为空"); - } - expr = typeParams.getExpr(); - // if (CollectionUtils.isEmpty(typeParams.getFields())) { - // throw new InvalidArgumentException("定义指标的字段列表参数不可为空"); - // } - if (!hasAggregateFunction(expr)) { - throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数"); - } - } - if (StringUtils.isBlank(expr)) { - throw new InvalidArgumentException("表达式不可为空"); - } - String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(metricReq.getName()); - if (StringUtils.isNotBlank(forbiddenCharacters)) { - throw new InvalidArgumentException( - String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters)); - } + }); } private static boolean hasAggregateFunction(String expr) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index 2ef90fb66..a7672d53a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -28,7 +28,7 @@ public class ModelConverter { public static ModelDO convert(ModelReq modelReq, User user) { ModelDO modelDO = new ModelDO(); - ModelDetail modelDetail = createModelDetail(modelReq); + ModelDetail modelDetail = convert(modelReq); modelReq.createdBy(user.getName()); BeanMapper.mapper(modelReq, modelDO); modelDO.setStatus(StatusEnum.ONLINE.getCode()); @@ -68,7 +68,7 @@ public class ModelConverter { } public static ModelDO convert(ModelDO modelDO, ModelReq modelReq, User user) { - ModelDetail modelDetail = updateModelDetail(modelReq); + ModelDetail modelDetail = convert(modelReq); BeanMapper.mapper(modelReq, modelDO); if (modelReq.getDrillDownDimensions() != null) { modelDO.setDrillDownDimensions( @@ -143,7 +143,7 @@ public class ModelConverter { } public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq, - String tableName) { + String tableName) { ModelReq modelReq = new ModelReq(); modelReq.setName(modelBuildReq.getName() != null ? modelBuildReq.getName() : tableName); modelReq.setBizName( @@ -169,7 +169,7 @@ public class ModelConverter { if (getIdentifyType(fieldType) != null) { Optional optional = modelDetail.getIdentifiers().stream().filter( - identify -> identify.getBizName().equals(semanticColumn.getColumnName())) + identify -> identify.getBizName().equals(semanticColumn.getColumnName())) .findAny(); if (optional.isEmpty()) { Identify identify = new Identify(semanticColumn.getName(), @@ -178,7 +178,7 @@ public class ModelConverter { } } else if (FieldType.measure.equals(fieldType)) { Optional optional = modelDetail.getMeasures().stream().filter( - measure -> measure.getBizName().equals(semanticColumn.getColumnName())) + measure -> measure.getBizName().equals(semanticColumn.getColumnName())) .findAny(); if (optional.isEmpty()) { Measure measure = new Measure(semanticColumn.getName(), @@ -188,7 +188,7 @@ public class ModelConverter { } } else { Optional optional = modelDetail.getDimensions().stream().filter( - dimension -> dimension.getBizName().equals(semanticColumn.getColumnName())) + dimension -> dimension.getBizName().equals(semanticColumn.getColumnName())) .findAny(); if (optional.isEmpty()) { Dimension dim = new Dimension(semanticColumn.getName(), @@ -288,18 +288,21 @@ public class ModelConverter { .collect(Collectors.toList()); } - private static ModelDetail createModelDetail(ModelReq modelReq) { + private static ModelDetail convert(ModelReq modelReq) { ModelDetail modelDetail = new ModelDetail(); List measures = modelReq.getModelDetail().getMeasures(); List dimensions = modelReq.getModelDetail().getDimensions(); List identifiers = modelReq.getModelDetail().getIdentifiers(); List fields = modelReq.getModelDetail().getFields(); + List fieldNames = fields.stream().map(Field::getFieldName).collect(Collectors.toList()); if (measures != null) { for (Measure measure : measures) { if (StringUtils.isNotBlank(measure.getBizName()) && StringUtils.isBlank(measure.getExpr())) { measure.setExpr(measure.getBizName()); + } + if (StringUtils.isNotBlank(measure.getBizName()) && !fieldNames.contains(measure.getBizName())) { fields.add(new Field(measure.getBizName(), "")); } } @@ -309,53 +312,24 @@ public class ModelConverter { if (StringUtils.isNotBlank(dimension.getBizName()) && StringUtils.isBlank(dimension.getExpr())) { dimension.setExpr(dimension.getBizName()); + } + if (StringUtils.isNotBlank(dimension.getBizName()) && !fieldNames.contains(dimension.getBizName())) { fields.add(new Field(dimension.getBizName(), "")); } } } if (identifiers != null) { for (Identify identify : identifiers) { - if (StringUtils.isNotBlank(identify.getBizName()) - && StringUtils.isBlank(identify.getName())) { + if (StringUtils.isNotBlank(identify.getBizName()) && StringUtils.isBlank(identify.getName())) { identify.setName(identify.getBizName()); } identify.setIsCreateDimension(1); - fields.add(new Field(identify.getBizName(), "")); - } - } - - BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); - return modelDetail; - } - - private static ModelDetail updateModelDetail(ModelReq modelReq) { - ModelDetail modelDetail = new ModelDetail(); - List measures = modelReq.getModelDetail().getMeasures(); - List dimensions = modelReq.getModelDetail().getDimensions(); - if (!CollectionUtils.isEmpty(dimensions)) { - for (Dimension dimension : dimensions) { - if (StringUtils.isNotBlank(dimension.getBizName()) - && StringUtils.isBlank(dimension.getExpr())) { - dimension.setExpr(dimension.getBizName()); + if (StringUtils.isNotBlank(identify.getBizName()) && !fieldNames.contains(identify.getBizName())) { + fields.add(new Field(identify.getBizName(), "")); } } } - if (measures == null) { - measures = Lists.newArrayList(); - } - for (Measure measure : measures) { - if (StringUtils.isBlank(measure.getBizName())) { - continue; - } - // Compatible with front-end tmp - String oriFieldName = - measure.getBizName().replaceFirst(modelReq.getBizName() + "_", ""); - measure.setExpr(oriFieldName); - if (!measure.getBizName().startsWith(modelReq.getBizName())) { - measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName)); - } - } BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); return modelDetail; } diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index 6f903887e..1f55544ad 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -3,26 +3,27 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + @@ -46,7 +47,7 @@ and ${criterion.condition} + open="(" separator=","> #{listItem} @@ -59,7 +60,7 @@ id - , name, biz_name, description, status, model_id, type, type_params, expr, datasource_id, + , name, biz_name, description, status, model_id, type, type_params, expr, created_at, created_by, updated_by, updated_at, semantic_type @@ -86,21 +87,36 @@ update s2_dimension - set name = #{dimension.name,jdbcType=VARCHAR}, - biz_name = #{dimension.bizName,jdbcType=VARCHAR}, - description = #{dimension.description,jdbcType=VARCHAR}, - status = #{dimension.status,jdbcType=INTEGER}, - model_id = #{dimension.modelId,jdbcType=BIGINT}, - type = #{dimension.type,jdbcType=VARCHAR}, - type_params = #{dimension.typeParams,jdbcType=VARCHAR}, - datasource_id = #{dimension.datasourceId,jdbcType=BIGINT}, - created_at = #{dimension.createdAt,jdbcType=TIMESTAMP}, - created_by = #{dimension.createdBy,jdbcType=VARCHAR}, - updated_by = #{dimension.updatedBy,jdbcType=VARCHAR}, - updated_at = #{dimension.updatedAt,jdbcType=TIMESTAMP}, - semantic_type = #{dimension.semanticType,jdbcType=VARCHAR}, - sensitive_level = #{dimension.sensitiveLevel,jdbcType=INTEGER}, - expr = #{dimension.expr,jdbcType=LONGVARCHAR} + set + name = #{dimension.name,jdbcType=VARCHAR}, + biz_name = + #{dimension.bizName,jdbcType=VARCHAR}, + + description = + #{dimension.description,jdbcType=VARCHAR}, + + status = #{dimension.status,jdbcType=INTEGER}, + model_id = #{dimension.modelId,jdbcType=BIGINT}, + type = #{dimension.type,jdbcType=VARCHAR}, + type_params = + #{dimension.typeParams,jdbcType=VARCHAR}, + + created_at = #{dimension.createdAt,jdbcType=TIMESTAMP}, + created_by = + #{dimension.createdBy,jdbcType=VARCHAR}, + + semantic_type = + #{dimension.semanticType,jdbcType=VARCHAR}, + + sensitive_level = + #{dimension.sensitiveLevel,jdbcType=INTEGER}, + + expr = #{dimension.expr,jdbcType=LONGVARCHAR}, + + updated_by = + #{dimension.updatedBy,jdbcType=VARCHAR}, + + updated_at = #{dimension.updatedAt,jdbcType=TIMESTAMP} where id = #{dimension.id,jdbcType=BIGINT} @@ -123,14 +139,14 @@ and model_id in + separator=","> #{model} and id in + separator=","> #{dimensionId} @@ -138,13 +154,13 @@ AND ( (name IN + separator=","> #{dimensionName} ) OR (biz_name IN + separator=","> #{dimensionName} ) ) diff --git a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml index 603dd5d81..93134644b 100644 --- a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml @@ -3,7 +3,7 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> + type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO"> @@ -23,7 +23,7 @@ + type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO"> @@ -46,7 +46,7 @@ and ${criterion.condition} + open="(" separator=","> #{listItem} @@ -97,6 +97,35 @@ + + + update s2_metric + set + name = #{metric.name,jdbcType=VARCHAR}, + biz_name = #{metric.bizName,jdbcType=VARCHAR}, + + description = + #{metric.description,jdbcType=VARCHAR}, + + status = #{metric.status,jdbcType=INTEGER}, + model_id = #{metric.modelId,jdbcType=BIGINT}, + type = #{metric.type,jdbcType=VARCHAR}, + type_params = + #{metric.typeParams,jdbcType=VARCHAR}, + + created_at = #{metric.createdAt,jdbcType=TIMESTAMP}, + created_by = + #{metric.createdBy,jdbcType=VARCHAR}, + + sensitive_level = #{metric.sensitiveLevel,jdbcType=INTEGER}, + updated_by = + #{metric.updatedBy,jdbcType=VARCHAR}, + + updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP} + where id = #{metric.id,jdbcType=BIGINT} + + + update s2_metric @@ -134,14 +163,14 @@ and model_id in + separator=","> #{model} and id in + separator=","> #{metricId} @@ -149,13 +178,13 @@ AND ( (name IN + separator=","> #{metricName} ) OR (biz_name IN + separator=","> #{metricName} ) ) diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/ClassDimensionTable.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/ClassDimensionTable.tsx index a90ef942a..776fe5406 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/ClassDimensionTable.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/ClassDimensionTable.tsx @@ -435,17 +435,17 @@ const ClassDimensionTable: React.FC = ({}) => { queryDimensionList({ ...filterParams, ...currentPagin }); }} toolBarRender={() => [ - , + // , = ({ message.error('请输入度量表达式'); return; } + + if (defineType === METRIC_DEFINE_TYPE.MEASURE) { + const { bizName, name, metricDefineByMeasureParams } = queryParams; + queryParams[queryParamsTypeParamsKey[METRIC_DEFINE_TYPE.MEASURE]].measures = + metricDefineByMeasureParams.measures.map((item: ISemantic.IMeasure) => { + return item.bizName === bizName && name ? { ...item, name } : item; + }); + } + if (!dataFormatType) { delete queryParams.dataFormat; } diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMeasuresFormTable.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMeasuresFormTable.tsx index 68092454b..961185441 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMeasuresFormTable.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/components/MetricMeasuresFormTable.tsx @@ -75,6 +75,14 @@ const MetricMeasuresFormTable: React.FC = ({ return tableData; }; + useEffect(() => { + if (measuresParams?.measures) { + setSelectedKeys(measuresParams.measures.map((item: ISemantic.IMeasure) => item.bizName)); + } + typeParams?.expr && setExprString(typeParams.expr); + }, [measuresParams, typeParams?.expr]); + + useEffect(() => { setTableData(getTableData()); }, [measuresList, measuresParams]); @@ -211,8 +219,8 @@ const MetricMeasuresFormTable: React.FC = ({ } - // tooltip="由于度量已自带聚合函数，因此通过度量创建指标时，表达式中无需再写聚合函数，如 - // 通过度量a和度量b来创建指标，由于建模的时候度量a和度量b被指定了聚合函数SUM，因此创建指标时表达式只需要写成 a+b, 而不需要带聚合函数" + // tooltip="由于度量已自带聚合函数，因此通过度量创建指标时，表达式中无需再写聚合函数，如 + // 通过度量a和度量b来创建指标，由于建模的时候度量a和度量b被指定了聚合函数SUM，因此创建指标时表达式只需要写成 a+b, 而不需要带聚合函数" >