[improvement](Headless) Adding change event to tags and integrating tag into embeddings (#792)

2025-12-10 19:51:00 +00:00 · 2024-03-06 16:13:20 +08:00
parent 475e071341
commit 3cccac58a1
8 changed files with 53 additions and 25 deletions
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMSqlParser.java
@@ -28,7 +28,7 @@ public class LLMSqlParser implements SemanticParser {
            return;
        }
        try {
-            //2.get modelId from queryCtx and chatCtx.
+            //2.get dataSetId from queryCtx and chatCtx.
            Long dataSetId = requestService.getDataSetId(queryCtx);
            if (dataSetId == null) {
                return;
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java
@@ -77,9 +77,7 @@ public class DataSetSchemaBuilder {

    private static Set<SchemaElement> getTags(DataSetSchemaResp resp) {
        Set<SchemaElement> tags = new HashSet<>();
-        List<TagResp> tagResps = resp.getTags();
-
-        for (TagResp tagResp : tagResps) {
+        for (TagResp tagResp : resp.getTags()) {
            SchemaElement element = SchemaElement.builder()
                    .dataSet(resp.getId())
                    .model(tagResp.getModelId())
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
@@ -43,7 +43,6 @@ public class RecommendServiceImpl implements RecommendService {
        if (Objects.isNull(limit) || limit <= 0) {
            limit = Long.MAX_VALUE;
        }
-        log.debug("limit:{}", limit);
        Long modelId = recommendReq.getModelId();
        if (Objects.isNull(modelId)) {
            return new RecommendResp();
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java
@@ -12,7 +12,7 @@ public enum DictWordType {

    DIMENSION("dimension"),

-    VALUE("value"),
+    VALUE("v"),

    VIEW("view"),

@@ -20,7 +20,7 @@ public enum DictWordType {

    NUMBER("m"),

-    TAG("tag"),
+    TAG("t"),

    SUFFIX("suffix");

@@ -45,7 +45,7 @@ public enum DictWordType {
                return dictWordType;
            }
        }
-        //domain
+        //view
        String[] natures = nature.split(DictWordType.NATURE_SPILT);
        if (natures.length == 2 && StringUtils.isNumeric(natures[1])) {
            return VIEW;
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java
@@ -1,13 +1,12 @@
 package com.tencent.supersonic.headless.api.pojo.response;

+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
-import lombok.Data;
-
 import javax.validation.constraints.NotNull;
-
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import lombok.Data;

@Data
 public class DictItemResp {
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
@@ -119,7 +119,6 @@ public class SchemaServiceImpl implements SchemaService {
    }

    public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
-
        MetaFilter metaFilter = new MetaFilter();
        metaFilter.setStatus(StatusEnum.ONLINE.getCode());
        metaFilter.setIds(filter.getDataSetIds());
@@ -141,7 +140,7 @@ public class SchemaServiceImpl implements SchemaService {

        TagFilter tagFilter = new TagFilter();
        tagFilter.setModelIds(modelIds);
-        List<TagResp> tagResps = tagService.getTags(tagFilter);
+        List<TagResp> tagRespList = tagService.getTags(tagFilter);

        List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
        for (Long dataSetId : dataSetRespMap.keySet()) {
@@ -160,8 +159,8 @@ public class SchemaServiceImpl implements SchemaService {
            dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
                    dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));

-            tagResps = TagConverter.filterByDataSet(tagResps, dataSetResp);
-            dataSetSchemaResp.setTags(tagResps);
+            tagRespList = TagConverter.filterByDataSet(tagRespList, dataSetResp);
+            dataSetSchemaResp.setTags(tagRespList);
            dataSetSchemaResps.add(dataSetSchemaResp);
        }
        fillStaticInfo(dataSetSchemaResps);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java
@@ -5,7 +5,11 @@ import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.DataEvent;
+import com.tencent.supersonic.common.pojo.DataItem;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
@@ -13,7 +17,6 @@ import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
-
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
@@ -25,7 +28,6 @@ import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -34,12 +36,12 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;

@Service
@@ -49,12 +51,14 @@ public class TagServiceImpl implements TagService {
    private final TagRepository tagRepository;
    private final ModelService modelService;
    private final CollectService collectService;
+    private ApplicationEventPublisher eventPublisher;

    public TagServiceImpl(TagRepository tagRepository, ModelService modelService,
-                          CollectService collectService) {
+            CollectService collectService, ApplicationEventPublisher eventPublisher) {
        this.tagRepository = tagRepository;
        this.modelService = modelService;
        this.collectService = collectService;
+        this.eventPublisher = eventPublisher;
    }

    @Override
@@ -72,21 +76,45 @@ public class TagServiceImpl implements TagService {
        } else {
            tagDO.setStatus(StatusEnum.ONLINE.getCode());
        }
-
        tagRepository.create(tagDO);
+        sendEventBatch(Lists.newArrayList(tagDO), EventType.ADD);
        return convert(tagDO);
    }

+    private void sendEventBatch(List<TagDO> tagDOS, EventType eventType) {
+        List<DataItem> dataItems = tagDOS.stream().map(this::getDataItem)
+                .collect(Collectors.toList());
+        eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType));
+    }
+
+    private void sendEvent(DataItem dataItem, EventType eventType) {
+        eventPublisher.publishEvent(new DataEvent(this,
+                Lists.newArrayList(dataItem), eventType));
+    }
+
+    private DataItem getDataItem(TagDO tagDO) {
+        return DataItem.builder().id(tagDO.getId() + Constants.UNDERLINE).name(tagDO.getName())
+                .bizName(tagDO.getBizName()).modelId(tagDO.getModelId() + Constants.UNDERLINE)
+                .type(TypeEnums.TAG).build();
+    }
+
    @Override
    public TagResp update(TagReq tagReq, User user) {
        if (Objects.isNull(tagReq.getId()) || tagReq.getId() <= 0) {
            throw new RuntimeException("id is empty");
        }
        TagDO tagDO = tagRepository.getTagById(tagReq.getId());
+        String oldName = tagDO.getName();
        tagDO = fillUpdateInfo(tagReq, tagDO);
        tagDO.setUpdatedBy(user.getName());
        tagDO.setUpdatedAt(new Date());
        tagRepository.update(tagDO);
+        if (!oldName.equals(tagReq.getName())) {
+            DataItem dataItem = getDataItem(tagDO);
+            dataItem.setName(oldName);
+            dataItem.setNewName(tagReq.getName());
+            sendEvent(getDataItem(tagDO), EventType.UPDATE);
+        }
        return convert(tagDO);
    }

@@ -100,6 +128,7 @@ public class TagServiceImpl implements TagService {
        tagDO.setUpdatedBy(user.getName());
        tagDO.setUpdatedAt(new Date());
        tagRepository.update(tagDO);
+        sendEventBatch(Lists.newArrayList(tagDO), EventType.DELETE);
    }

    @Override
@@ -142,7 +171,7 @@ public class TagServiceImpl implements TagService {
        }

        PageInfo<TagDO> tagDOPageInfo = PageHelper.startPage(tagFilterPage.getCurrent(),
-                tagFilterPage.getPageSize())
+                        tagFilterPage.getPageSize())
                .doSelectPageInfo(() -> getTags(tagFilter));
        PageInfo<TagResp> pageInfo = new PageInfo<>();
        BeanUtils.copyProperties(tagDOPageInfo, pageInfo);
@@ -173,8 +202,12 @@ public class TagServiceImpl implements TagService {
        });

        tagRepository.batchUpdateStatus(tagDOList);
-        // todo  sendEventBatch
-
+        if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
+                || StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
+            sendEventBatch(tagDOList, EventType.DELETE);
+        } else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
+            sendEventBatch(tagDOList, EventType.ADD);
+        }
        return true;
    }

--- a/launchers/standalone/src/main/resources/application-local.yaml
+++ b/launchers/standalone/src/main/resources/application-local.yaml
@@ -61,7 +61,7 @@ s2:
    chat-model:
      provider: open_ai
      openai:
-        api-key: app_key
+        api-key: api_key
        model-name: gpt-3.5-turbo-16k
        temperature: 0.0
        timeout: PT60S