(improvement) optimize schema data change monitoring (#333)

Co-authored-by: jolunoluo

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java

@@ -28,7 +28,7 @@ public class SchemaElement implements Serializable {
 
     private String defaultAgg;
 
-    private int order;
+    private double order;
 
     @Override
     public boolean equals(Object o) {

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java

@@ -54,7 +54,11 @@ public class SemanticParseInfo {
         @Override
         public int compare(SchemaElement o1, SchemaElement o2) {
             if (o1.getOrder() != o2.getOrder()) {
-                return o1.getOrder() - o2.getOrder();
+                if (o1.getOrder() < o2.getOrder()) {
+                    return -1;
+                } else {
+                    return 1;
+                }
             }
             int len1 = o1.getName().length();
             int len2 = o2.getName().length();

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java

@@ -47,7 +47,9 @@ public class EmbeddingMapper extends BaseMapper {
             long modelId = Long.parseLong(modelIdStr);
 
             schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
-
+            if (schemaElement == null) {
+                continue;
+            }
             SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
                     .element(schemaElement)
                     .frequency(BaseWordBuilder.DEFAULT_FREQUENCY)

chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java

@@ -109,6 +109,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
 
         for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
             SchemaElement element = schemaMatch.getElement();
+            element.setOrder(1 - schemaMatch.getSimilarity());
             switch (element.getType()) {
                 case ID:
                     SchemaElement entityElement = modelSchema.getElement(SchemaElementType.ENTITY, element.getId());

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java

@@ -230,6 +230,9 @@ public class ConfigServiceImpl implements ConfigService {
         BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
 
         ModelSchema modelSchema = semanticService.getModelSchema(modelId);
+        if (modelSchema == null) {
+            return chatConfigRich;
+        }
         chatConfigRich.setBizName(modelSchema.getModel().getBizName());
         chatConfigRich.setModelName(modelSchema.getModel().getName());
 

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java

@@ -51,6 +51,9 @@ public class RecommendServiceImpl implements RecommendService {
             return new RecommendResp();
         }
         ModelSchema modelSchema = semanticService.getModelSchema(modelId);
+        if (Objects.isNull(modelSchema)) {
+            return new RecommendResp();
+        }
         List<Long> drillDownDimensions = Lists.newArrayList();
         Set<SchemaElement> metricElements = modelSchema.getMetrics();
         if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) {

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java

@@ -5,21 +5,29 @@ import com.tencent.supersonic.common.pojo.DataEvent;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.knowledge.dictionary.DictWord;
+import com.tencent.supersonic.knowledge.service.SchemaService;
 import com.tencent.supersonic.knowledge.utils.HanlpHelper;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationListener;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
 @Component
 @Slf4j
-public class DictUpdateListener implements ApplicationListener<DataEvent> {
+public class SchemaDictUpdateListener implements ApplicationListener<DataEvent> {
 
+    @Autowired
+    private SchemaService schemaService;
+
+    @Async
     @Override
     public void onApplicationEvent(DataEvent dataEvent) {
         if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
             return;
         }
+        schemaService.getCache().invalidateAll();
         dataEvent.getDataItems().forEach(dataItem -> {
             DictWord dictWord = new DictWord();
             dictWord.setWord(dataItem.getName());

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java

@@ -18,11 +18,11 @@ public class SchemaService {
 
 
     public static final String ALL_CACHE = "all";
-    private static final Integer META_CACHE_TIME = 2;
+    private static final Integer META_CACHE_TIME = 30;
     private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
 
     private LoadingCache<String, SemanticSchema> cache = CacheBuilder.newBuilder()
-            .expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
+            .expireAfterWrite(META_CACHE_TIME, TimeUnit.SECONDS)
             .build(
                     new CacheLoader<String, SemanticSchema>() {
                         @Override

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java

@@ -230,6 +230,17 @@ public class DimensionServiceImpl implements DimensionService {
         return dimensionResps;
     }
 
+    @Override
+    public List<DataItem> getDataItems(Long modelId) {
+        DimensionFilter metaFilter = new DimensionFilter();
+        metaFilter.setModelIds(Lists.newArrayList(modelId));
+        List<DimensionDO> dimensionDOS = queryDimension(metaFilter);
+        if (CollectionUtils.isEmpty(dimensionDOS)) {
+            return Lists.newArrayList();
+        }
+        return dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList());
+    }
+
     @Override
     public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
         String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(),
@@ -333,5 +344,11 @@ public class DimensionServiceImpl implements DimensionService {
                 Lists.newArrayList(dataItem), eventType));
     }
 
+    private DataItem getDataItem(DimensionDO dimensionDO) {
+        return DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName())
+                .bizName(dimensionDO.getBizName())
+                .modelId(dimensionDO.getModelId()).type(TypeEnums.DIMENSION).build();
+    }
+
 
 }

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java

@@ -43,7 +43,6 @@ import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -60,17 +59,18 @@ public class MetricServiceImpl implements MetricService {
 
     private ChatGptHelper chatGptHelper;
 
-    @Autowired
     private ApplicationEventPublisher eventPublisher;
 
     public MetricServiceImpl(MetricRepository metricRepository,
                              ModelService modelService,
                              DomainService domainService,
-                             ChatGptHelper chatGptHelper) {
+                             ChatGptHelper chatGptHelper,
+                             ApplicationEventPublisher eventPublisher) {
         this.domainService = domainService;
         this.metricRepository = metricRepository;
         this.modelService = modelService;
         this.chatGptHelper = chatGptHelper;
+        this.eventPublisher = eventPublisher;
     }
 
     @Override
@@ -276,6 +276,17 @@ public class MetricServiceImpl implements MetricService {
         return modelResp.getDrillDownDimensions();
     }
 
+    @Override
+    public List<DataItem> getDataItems(Long modelId) {
+        MetricFilter metaFilter = new MetricFilter();
+        metaFilter.setModelIds(Lists.newArrayList(modelId));
+        List<MetricDO> metricDOS = queryMetric(metaFilter);
+        if (CollectionUtils.isEmpty(metricDOS)) {
+            return Lists.newArrayList();
+        }
+        return metricDOS.stream().map(this::getDataItem).collect(Collectors.toList());
+    }
+
     private void checkParam(MetricReq metricReq) {
         MetricTypeParams typeParams = metricReq.getTypeParams();
         List<Measure> measures = typeParams.getMeasures();

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java

@@ -3,7 +3,9 @@ package com.tencent.supersonic.semantic.model.application;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.service.UserService;
+import com.tencent.supersonic.common.pojo.DataEvent;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
@@ -43,6 +45,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -59,12 +62,13 @@ public class ModelServiceImpl implements ModelService {
     private final UserService userService;
     private final DatabaseService databaseService;
     private final Catalog catalog;
+    private ApplicationEventPublisher eventPublisher;
 
     public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService,
             @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService,
             @Lazy DomainService domainService, UserService userService,
             @Lazy DatabaseService databaseService,
-            @Lazy Catalog catalog) {
+            @Lazy Catalog catalog, ApplicationEventPublisher eventPublisher) {
         this.modelRepository = modelRepository;
         this.metricService = metricService;
         this.dimensionService = dimensionService;
@@ -73,6 +77,7 @@ public class ModelServiceImpl implements ModelService {
         this.userService = userService;
         this.databaseService = databaseService;
         this.catalog = catalog;
+        this.eventPublisher = eventPublisher;
     }
 
     @Override
@@ -86,6 +91,7 @@ public class ModelServiceImpl implements ModelService {
     public void updateModel(ModelReq modelReq, User user) {
         ModelDO modelDO = getModelDO(modelReq.getId());
         modelReq.updatedBy(user.getName());
+        int oldStatus = modelDO.getStatus();
         BeanMapper.mapper(modelReq, modelDO);
         if (modelReq.getEntity() != null) {
             modelDO.setEntity(JsonUtil.toString(modelReq.getEntity()));
@@ -94,6 +100,29 @@ public class ModelServiceImpl implements ModelService {
             modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions()));
         }
         modelRepository.updateModel(modelDO);
+        statusPublish(oldStatus, modelDO);
+    }
+
+    private void statusPublish(Integer oldStatus, ModelDO modelDO) {
+        if (oldStatus.equals(modelDO.getStatus())) {
+            return;
+        }
+        if (oldStatus.equals(StatusEnum.ONLINE.getCode())
+                && modelDO.getStatus().equals(StatusEnum.OFFLINE.getCode())) {
+            publishEvent(EventType.DELETE, modelDO);
+        } else if (oldStatus.equals(StatusEnum.OFFLINE.getCode())
+                && modelDO.getStatus().equals(StatusEnum.ONLINE.getCode())) {
+            publishEvent(EventType.ADD, modelDO);
+        }
+    }
+
+    private void publishEvent(EventType eventType, ModelDO modelDO) {
+        eventPublisher.publishEvent(
+                new DataEvent(this, metricService.getDataItems(modelDO.getId()),
+                        eventType));
+        eventPublisher.publishEvent(
+                new DataEvent(this, dimensionService.getDataItems(modelDO.getId()),
+                        eventType));
     }
 
     @Override

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain;
 
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.DataItem;
 import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
 import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
 import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
@@ -31,6 +32,8 @@ public interface DimensionService {
 
     void deleteDimension(Long id, User user);
 
+    List<DataItem> getDataItems(Long modelId);
+
     List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user);
 
     List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain;
 
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.DataItem;
 import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
 import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
 import com.tencent.supersonic.semantic.api.model.request.MetricReq;
@@ -34,4 +35,6 @@ public interface MetricService {
     Set<String> getMetricTags();
 
     List<DrillDownDimension> getDrillDownDimension(Long metricId);
+
+    List<DataItem> getDataItems(Long modelId);
 }

semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml

@@ -151,6 +151,13 @@
                 #{model}
             </foreach>
         </if>
+        <if test="ids != null and ids.size >0">
+            and id in
+            <foreach collection="ids" index="index" item="id" open="(" close=")"
+                     separator=",">
+                #{id}
+            </foreach>
+        </if>
         <if test="createdBy != null">
             and created_by = #{createdBy}
         </if>