From aa6c658a9ad5781bfc5c1b19ef2ddc73401d9c88 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Mon, 6 Nov 2023 22:11:56 +0800 Subject: [PATCH] (improvement) optimize schema data change monitoring (#333) Co-authored-by: jolunoluo --- .../chat/api/pojo/SchemaElement.java | 2 +- .../chat/api/pojo/SemanticParseInfo.java | 6 +++- .../chat/mapper/EmbeddingMapper.java | 4 ++- .../chat/query/rule/RuleSemanticQuery.java | 1 + .../chat/service/impl/ConfigServiceImpl.java | 3 ++ .../service/impl/RecommendServiceImpl.java | 3 ++ ...ner.java => SchemaDictUpdateListener.java} | 10 +++++- .../knowledge/service/SchemaService.java | 4 +-- .../application/DimensionServiceImpl.java | 17 ++++++++++ .../model/application/MetricServiceImpl.java | 17 ++++++++-- .../model/application/ModelServiceImpl.java | 31 ++++++++++++++++++- .../model/domain/DimensionService.java | 3 ++ .../semantic/model/domain/MetricService.java | 3 ++ .../mapper/custom/DimensionDOCustomMapper.xml | 7 +++++ 14 files changed, 101 insertions(+), 10 deletions(-) rename chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/{DictUpdateListener.java => SchemaDictUpdateListener.java} (83%) diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java index 97a43dda6..eb929e3d4 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java @@ -28,7 +28,7 @@ public class SchemaElement implements Serializable { private String defaultAgg; - private int order; + private double order; @Override public boolean equals(Object o) { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index f4dd3bb03..79e8df5cd 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -54,7 +54,11 @@ public class SemanticParseInfo { @Override public int compare(SchemaElement o1, SchemaElement o2) { if (o1.getOrder() != o2.getOrder()) { - return o1.getOrder() - o2.getOrder(); + if (o1.getOrder() < o2.getOrder()) { + return -1; + } else { + return 1; + } } int len1 = o1.getName().length(); int len2 = o2.getName().length(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java index e95bc8db8..e6665db9a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java @@ -47,7 +47,9 @@ public class EmbeddingMapper extends BaseMapper { long modelId = Long.parseLong(modelIdStr); schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId); - + if (schemaElement == null) { + continue; + } SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() .element(schemaElement) .frequency(BaseWordBuilder.DEFAULT_FREQUENCY) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index 06d0a62bd..c22ae8a22 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -109,6 +109,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) { SchemaElement element = schemaMatch.getElement(); + element.setOrder(1 - schemaMatch.getSimilarity()); switch (element.getType()) { case ID: SchemaElement entityElement = modelSchema.getElement(SchemaElementType.ENTITY, element.getId()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java index ad13f7279..4bde4f722 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java @@ -230,6 +230,9 @@ public class ConfigServiceImpl implements ConfigService { BeanUtils.copyProperties(chatConfigResp, chatConfigRich); ModelSchema modelSchema = semanticService.getModelSchema(modelId); + if (modelSchema == null) { + return chatConfigRich; + } chatConfigRich.setBizName(modelSchema.getModel().getBizName()); chatConfigRich.setModelName(modelSchema.getModel().getName()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java index cb554e795..ef5cb28b3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java @@ -51,6 +51,9 @@ public class RecommendServiceImpl implements RecommendService { return new RecommendResp(); } ModelSchema modelSchema = semanticService.getModelSchema(modelId); + if (Objects.isNull(modelSchema)) { + return new RecommendResp(); + } List drillDownDimensions = Lists.newArrayList(); Set metricElements = modelSchema.getMetrics(); if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) { diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java similarity index 83% rename from chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java rename to chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java index 97cf47f4b..bc2374a92 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java @@ -5,21 +5,29 @@ import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.knowledge.dictionary.DictWord; +import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @Component @Slf4j -public class DictUpdateListener implements ApplicationListener { +public class SchemaDictUpdateListener implements ApplicationListener { + @Autowired + private SchemaService schemaService; + + @Async @Override public void onApplicationEvent(DataEvent dataEvent) { if (CollectionUtils.isEmpty(dataEvent.getDataItems())) { return; } + schemaService.getCache().invalidateAll(); dataEvent.getDataItems().forEach(dataItem -> { DictWord dictWord = new DictWord(); dictWord.setWord(dataItem.getName()); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java index 56b74e3d6..9ce75ad34 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java @@ -18,11 +18,11 @@ public class SchemaService { public static final String ALL_CACHE = "all"; - private static final Integer META_CACHE_TIME = 2; + private static final Integer META_CACHE_TIME = 30; private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); private LoadingCache cache = CacheBuilder.newBuilder() - .expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES) + .expireAfterWrite(META_CACHE_TIME, TimeUnit.SECONDS) .build( new CacheLoader() { @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 9c37a70ae..7e772bd44 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -230,6 +230,17 @@ public class DimensionServiceImpl implements DimensionService { return dimensionResps; } + @Override + public List getDataItems(Long modelId) { + DimensionFilter metaFilter = new DimensionFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); + List dimensionDOS = queryDimension(metaFilter); + if (CollectionUtils.isEmpty(dimensionDOS)) { + return Lists.newArrayList(); + } + return dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList()); + } + @Override public List mockAlias(DimensionReq dimensionReq, String mockType, User user) { String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(), @@ -333,5 +344,11 @@ public class DimensionServiceImpl implements DimensionService { Lists.newArrayList(dataItem), eventType)); } + private DataItem getDataItem(DimensionDO dimensionDO) { + return DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName()) + .bizName(dimensionDO.getBizName()) + .modelId(dimensionDO.getModelId()).type(TypeEnums.DIMENSION).build(); + } + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index f9ff7469d..a230f6228 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -43,7 +43,6 @@ import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -60,17 +59,18 @@ public class MetricServiceImpl implements MetricService { private ChatGptHelper chatGptHelper; - @Autowired private ApplicationEventPublisher eventPublisher; public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, DomainService domainService, - ChatGptHelper chatGptHelper) { + ChatGptHelper chatGptHelper, + ApplicationEventPublisher eventPublisher) { this.domainService = domainService; this.metricRepository = metricRepository; this.modelService = modelService; this.chatGptHelper = chatGptHelper; + this.eventPublisher = eventPublisher; } @Override @@ -276,6 +276,17 @@ public class MetricServiceImpl implements MetricService { return modelResp.getDrillDownDimensions(); } + @Override + public List getDataItems(Long modelId) { + MetricFilter metaFilter = new MetricFilter(); + metaFilter.setModelIds(Lists.newArrayList(modelId)); + List metricDOS = queryMetric(metaFilter); + if (CollectionUtils.isEmpty(metricDOS)) { + return Lists.newArrayList(); + } + return metricDOS.stream().map(this::getDataItem).collect(Collectors.toList()); + } + private void checkParam(MetricReq metricReq) { MetricTypeParams typeParams = metricReq.getTypeParams(); List measures = typeParams.getMeasures(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 3ae4975b3..9eaeb0a62 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -3,7 +3,9 @@ package com.tencent.supersonic.semantic.model.application; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; +import com.tencent.supersonic.common.pojo.DataEvent; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; @@ -43,6 +45,7 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -59,12 +62,13 @@ public class ModelServiceImpl implements ModelService { private final UserService userService; private final DatabaseService databaseService; private final Catalog catalog; + private ApplicationEventPublisher eventPublisher; public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService, @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService, @Lazy DomainService domainService, UserService userService, @Lazy DatabaseService databaseService, - @Lazy Catalog catalog) { + @Lazy Catalog catalog, ApplicationEventPublisher eventPublisher) { this.modelRepository = modelRepository; this.metricService = metricService; this.dimensionService = dimensionService; @@ -73,6 +77,7 @@ public class ModelServiceImpl implements ModelService { this.userService = userService; this.databaseService = databaseService; this.catalog = catalog; + this.eventPublisher = eventPublisher; } @Override @@ -86,6 +91,7 @@ public class ModelServiceImpl implements ModelService { public void updateModel(ModelReq modelReq, User user) { ModelDO modelDO = getModelDO(modelReq.getId()); modelReq.updatedBy(user.getName()); + int oldStatus = modelDO.getStatus(); BeanMapper.mapper(modelReq, modelDO); if (modelReq.getEntity() != null) { modelDO.setEntity(JsonUtil.toString(modelReq.getEntity())); @@ -94,6 +100,29 @@ public class ModelServiceImpl implements ModelService { modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions())); } modelRepository.updateModel(modelDO); + statusPublish(oldStatus, modelDO); + } + + private void statusPublish(Integer oldStatus, ModelDO modelDO) { + if (oldStatus.equals(modelDO.getStatus())) { + return; + } + if (oldStatus.equals(StatusEnum.ONLINE.getCode()) + && modelDO.getStatus().equals(StatusEnum.OFFLINE.getCode())) { + publishEvent(EventType.DELETE, modelDO); + } else if (oldStatus.equals(StatusEnum.OFFLINE.getCode()) + && modelDO.getStatus().equals(StatusEnum.ONLINE.getCode())) { + publishEvent(EventType.ADD, modelDO); + } + } + + private void publishEvent(EventType eventType, ModelDO modelDO) { + eventPublisher.publishEvent( + new DataEvent(this, metricService.getDataItems(modelDO.getId()), + eventType)); + eventPublisher.publishEvent( + new DataEvent(this, dimensionService.getDataItems(modelDO.getId()), + eventType)); } @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java index 3e0bda9e8..8d05a125e 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.request.DimensionReq; import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; @@ -31,6 +32,8 @@ public interface DimensionService { void deleteDimension(Long id, User user); + List getDataItems(Long modelId); + List mockAlias(DimensionReq dimensionReq, String mockType, User user); List mockDimensionValueAlias(DimensionReq dimensionReq, User user); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java index 8e346a01a..30b0ed48b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq; import com.tencent.supersonic.semantic.api.model.request.MetricReq; @@ -34,4 +35,6 @@ public interface MetricService { Set getMetricTags(); List getDrillDownDimension(Long metricId); + + List getDataItems(Long modelId); } diff --git a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index fd5e1bf82..6b4338fab 100644 --- a/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -151,6 +151,13 @@ #{model} + + and id in + + #{id} + + and created_by = #{createdBy}