[improvement](Headless) Adding change event to tags and integrating tag into embeddings (#792)

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java

@@ -28,7 +28,7 @@ public class LLMSqlParser implements SemanticParser {
             return;
         }
         try {
-            //2.get modelId from queryCtx and chatCtx.
+            //2.get dataSetId from queryCtx and chatCtx.
             Long dataSetId = requestService.getDataSetId(queryCtx);
             if (dataSetId == null) {
                 return;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java

@@ -77,9 +77,7 @@ public class DataSetSchemaBuilder {
 
     private static Set<SchemaElement> getTags(DataSetSchemaResp resp) {
         Set<SchemaElement> tags = new HashSet<>();
-        List<TagResp> tagResps = resp.getTags();
+        for (TagResp tagResp : resp.getTags()) {
-
-        for (TagResp tagResp : tagResps) {
             SchemaElement element = SchemaElement.builder()
                     .dataSet(resp.getId())
                     .model(tagResp.getModelId())

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java

@@ -43,7 +43,6 @@ public class RecommendServiceImpl implements RecommendService {
         if (Objects.isNull(limit) || limit <= 0) {
             limit = Long.MAX_VALUE;
         }
-        log.debug("limit:{}", limit);
         Long modelId = recommendReq.getModelId();
         if (Objects.isNull(modelId)) {
             return new RecommendResp();

common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java

@@ -12,7 +12,7 @@ public enum DictWordType {
 
     DIMENSION("dimension"),
 
-    VALUE("value"),
+    VALUE("v"),
 
     VIEW("view"),
 
@@ -20,7 +20,7 @@ public enum DictWordType {
 
     NUMBER("m"),
 
-    TAG("tag"),
+    TAG("t"),
 
     SUFFIX("suffix");
 
@@ -45,7 +45,7 @@ public enum DictWordType {
                 return dictWordType;
             }
         }
-        //domain
+        //view
         String[] natures = nature.split(DictWordType.NATURE_SPILT);
         if (natures.length == 2 && StringUtils.isNumeric(natures[1])) {
             return VIEW;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java

@@ -1,13 +1,12 @@
 package com.tencent.supersonic.headless.api.pojo.response;
 
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
-import lombok.Data;
-
 import javax.validation.constraints.NotNull;
-
+import lombok.Data;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
 
 @Data
 public class DictItemResp {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java

@@ -119,7 +119,6 @@ public class SchemaServiceImpl implements SchemaService {
     }
 
     public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
-
         MetaFilter metaFilter = new MetaFilter();
         metaFilter.setStatus(StatusEnum.ONLINE.getCode());
         metaFilter.setIds(filter.getDataSetIds());
@@ -141,7 +140,7 @@ public class SchemaServiceImpl implements SchemaService {
 
         TagFilter tagFilter = new TagFilter();
         tagFilter.setModelIds(modelIds);
-        List<TagResp> tagResps = tagService.getTags(tagFilter);
+        List<TagResp> tagRespList = tagService.getTags(tagFilter);
 
         List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
         for (Long dataSetId : dataSetRespMap.keySet()) {
@@ -160,8 +159,8 @@ public class SchemaServiceImpl implements SchemaService {
             dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
                     dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
 
-            tagResps = TagConverter.filterByDataSet(tagResps, dataSetResp);
+            tagRespList = TagConverter.filterByDataSet(tagRespList, dataSetResp);
-            dataSetSchemaResp.setTags(tagResps);
+            dataSetSchemaResp.setTags(tagRespList);
             dataSetSchemaResps.add(dataSetSchemaResp);
         }
         fillStaticInfo(dataSetSchemaResps);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java

@@ -5,7 +5,11 @@ import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.DataEvent;
+import com.tencent.supersonic.common.pojo.DataItem;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
@@ -13,7 +17,6 @@ import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
-
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
@@ -25,7 +28,6 @@ import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -34,12 +36,12 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
 
 @Service
@@ -49,12 +51,14 @@ public class TagServiceImpl implements TagService {
     private final TagRepository tagRepository;
     private final ModelService modelService;
     private final CollectService collectService;
+    private ApplicationEventPublisher eventPublisher;
 
     public TagServiceImpl(TagRepository tagRepository, ModelService modelService,
-                          CollectService collectService) {
+            CollectService collectService, ApplicationEventPublisher eventPublisher) {
         this.tagRepository = tagRepository;
         this.modelService = modelService;
         this.collectService = collectService;
+        this.eventPublisher = eventPublisher;
     }
 
     @Override
@@ -72,21 +76,45 @@ public class TagServiceImpl implements TagService {
         } else {
             tagDO.setStatus(StatusEnum.ONLINE.getCode());
         }
-
         tagRepository.create(tagDO);
+        sendEventBatch(Lists.newArrayList(tagDO), EventType.ADD);
         return convert(tagDO);
     }
 
+    private void sendEventBatch(List<TagDO> tagDOS, EventType eventType) {
+        List<DataItem> dataItems = tagDOS.stream().map(this::getDataItem)
+                .collect(Collectors.toList());
+        eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType));
+    }
+
+    private void sendEvent(DataItem dataItem, EventType eventType) {
+        eventPublisher.publishEvent(new DataEvent(this,
+                Lists.newArrayList(dataItem), eventType));
+    }
+
+    private DataItem getDataItem(TagDO tagDO) {
+        return DataItem.builder().id(tagDO.getId() + Constants.UNDERLINE).name(tagDO.getName())
+                .bizName(tagDO.getBizName()).modelId(tagDO.getModelId() + Constants.UNDERLINE)
+                .type(TypeEnums.TAG).build();
+    }
+
     @Override
     public TagResp update(TagReq tagReq, User user) {
         if (Objects.isNull(tagReq.getId()) || tagReq.getId() <= 0) {
             throw new RuntimeException("id is empty");
         }
         TagDO tagDO = tagRepository.getTagById(tagReq.getId());
+        String oldName = tagDO.getName();
         tagDO = fillUpdateInfo(tagReq, tagDO);
         tagDO.setUpdatedBy(user.getName());
         tagDO.setUpdatedAt(new Date());
         tagRepository.update(tagDO);
+        if (!oldName.equals(tagReq.getName())) {
+            DataItem dataItem = getDataItem(tagDO);
+            dataItem.setName(oldName);
+            dataItem.setNewName(tagReq.getName());
+            sendEvent(getDataItem(tagDO), EventType.UPDATE);
+        }
         return convert(tagDO);
     }
 
@@ -100,6 +128,7 @@ public class TagServiceImpl implements TagService {
         tagDO.setUpdatedBy(user.getName());
         tagDO.setUpdatedAt(new Date());
         tagRepository.update(tagDO);
+        sendEventBatch(Lists.newArrayList(tagDO), EventType.DELETE);
     }
 
     @Override
@@ -173,8 +202,12 @@ public class TagServiceImpl implements TagService {
         });
 
         tagRepository.batchUpdateStatus(tagDOList);
-        // todo  sendEventBatch
+        if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
-
+                || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
+            sendEventBatch(tagDOList, EventType.DELETE);
+        } else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
+            sendEventBatch(tagDOList, EventType.ADD);
+        }
         return true;
     }
 

launchers/standalone/src/main/resources/application-local.yaml

@@ -61,7 +61,7 @@ s2:
     chat-model:
       provider: open_ai
       openai:
-        api-key: app_key
+        api-key: api_key
         model-name: gpt-3.5-turbo-16k
         temperature: 0.0
         timeout: PT60S