(improvement)(headless) transfer term nature modelId to viewId before providing it to chat and put the modelId of metadata into the dict word instead of viewId (#739)

Co-authored-by: jolunoluo
2026-04-24 08:24:19 +08:00 · 2024-02-23 10:29:21 +08:00
parent 16643e8d75
commit e95a528219
52 changed files with 456 additions and 245 deletions
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java
@@ -45,7 +45,8 @@ public class GroupByCorrector extends BaseSemanticCorrector {
        ViewResp viewResp = viewService.getView(viewId);
        List<Long> modelIds = viewResp.getViewDetail().getViewModelConfigs().stream().map(config -> config.getId())
                .collect(Collectors.toList());
-        MetaFilter metaFilter = new MetaFilter(modelIds);
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setIds(modelIds);
        List<ModelResp> modelRespList = modelService.getModelList(metaFilter);
        for (ModelResp modelResp : modelRespList) {
            List<Dim> dimList = modelResp.getModelDetail().getDimensions();
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java
@@ -10,9 +10,10 @@ import com.tencent.supersonic.headless.api.pojo.response.S2Term;
 import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
 import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
 import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
+import lombok.extern.slf4j.Slf4j;
 import java.util.List;
 import java.util.Objects;
-import lombok.extern.slf4j.Slf4j;

 /***
 * A mapper that recognizes schema elements with vector embedding.
@@ -24,7 +25,8 @@ public class EmbeddingMapper extends BaseMapper {
    public void doMap(QueryContext queryContext) {
        //1. query from embedding by queryText
        String queryText = queryContext.getQueryText();
-        List<S2Term> terms = HanlpHelper.getTerms(queryText);
+        KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class);
+        List<S2Term> terms = knowledgeService.getTerms(queryText);

        EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
        List<EmbeddingResult> matchResults = matchStrategy.getMatches(queryContext, terms);
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java
@@ -1,36 +1,27 @@
 package com.tencent.supersonic.chat.core.parser.sql.llm;

-import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
-import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
+import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.util.DatePeriodEnum;
 import com.tencent.supersonic.common.util.DateUtils;
+import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
+
 import java.util.Objects;

 public class S2SqlDateHelper {

-    public static String getReferenceDate(QueryContext queryContext, Long modelId) {
+    public static String getReferenceDate(QueryContext queryContext, Long viewId) {
        String defaultDate = DateUtils.getBeforeDate(0);
-        if (Objects.isNull(modelId)) {
+        if (Objects.isNull(viewId)) {
            return defaultDate;
        }
-        ChatConfigFilter filter = new ChatConfigFilter();
-        filter.setModelId(modelId);
-        ChatConfigRichResp chatConfigRichResp = queryContext.getModelIdToChatRichConfig().get(modelId);
-
-        if (Objects.isNull(chatConfigRichResp)) {
+        ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId);
+        if (viewSchema == null || viewSchema.getTagTypeTimeDefaultConfig() == null) {
            return defaultDate;
        }
-        if (Objects.isNull(chatConfigRichResp.getChatDetailRichConfig()) || Objects.isNull(
-                chatConfigRichResp.getChatDetailRichConfig().getChatDefaultConfig())) {
-            return defaultDate;
-        }
-
-        ChatDefaultRichConfigResp chatDefaultConfig = chatConfigRichResp.getChatDetailRichConfig()
-                .getChatDefaultConfig();
-        Integer unit = chatDefaultConfig.getUnit();
-        String period = chatDefaultConfig.getPeriod();
+        TimeDefaultConfig tagTypeTimeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig();
+        Integer unit = tagTypeTimeDefaultConfig.getUnit();
+        String period = tagTypeTimeDefaultConfig.getPeriod();
        if (Objects.nonNull(unit)) {
            // If the unit is set to less than 0, then do not add relative date.
            if (unit < 0) {
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java
@@ -44,6 +44,7 @@ public class ViewSchemaBuilder {

            SchemaElement metricToAdd = SchemaElement.builder()
                    .view(resp.getId())
+                    .model(metric.getModelId())
                    .id(metric.getId())
                    .name(metric.getName())
                    .bizName(metric.getBizName())
@@ -84,6 +85,7 @@ public class ViewSchemaBuilder {
            }
            SchemaElement dimToAdd = SchemaElement.builder()
                    .view(resp.getId())
+                    .model(dim.getModelId())
                    .id(dim.getId())
                    .name(dim.getName())
                    .bizName(dim.getBizName())
@@ -96,6 +98,7 @@ public class ViewSchemaBuilder {

            SchemaElement dimValueToAdd = SchemaElement.builder()
                    .view(resp.getId())
+                    .model(dim.getModelId())
                    .id(dim.getId())
                    .name(dim.getName())
                    .bizName(dim.getBizName())
@@ -107,6 +110,7 @@ public class ViewSchemaBuilder {
            if (dim.getIsTag() == 1) {
                SchemaElement tagToAdd = SchemaElement.builder()
                        .view(resp.getId())
+                        .model(dim.getModelId())
                        .id(dim.getId())
                        .name(dim.getName())
                        .bizName(dim.getBizName())
@@ -126,6 +130,7 @@ public class ViewSchemaBuilder {
        if (dim != null) {
            SchemaElement entity = SchemaElement.builder()
                    .view(resp.getId())
+                    .model(dim.getModelId())
                    .id(dim.getId())
                    .name(dim.getName())
                    .bizName(dim.getBizName())
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java
@@ -38,11 +38,11 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
        queryResult.setRecommendedDimensions(dimensionRecommended);
    }

-    private List<SchemaElement> getDimensions(Long metricId, Long modelId) {
+    private List<SchemaElement> getDimensions(Long metricId, Long viewId) {
        SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
-        ViewSchema modelSchema = semanticService.getModelSchema(modelId);
+        ViewSchema viewSchema = semanticService.getViewSchema(viewId);
        List<Long> drillDownDimensions = Lists.newArrayList();
-        Set<SchemaElement> metricElements = modelSchema.getMetrics();
+        Set<SchemaElement> metricElements = viewSchema.getMetrics();
        if (!CollectionUtils.isEmpty(metricElements)) {
            Optional<SchemaElement> metric = metricElements.stream().filter(schemaElement ->
                            metricId.equals(schemaElement.getId())
@@ -54,7 +54,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
            }
        }
        final List<Long> drillDownDimensionsFinal = drillDownDimensions;
-        return modelSchema.getDimensions().stream()
+        return viewSchema.getDimensions().stream()
                .filter(dim -> filterDimension(drillDownDimensionsFinal, dim))
                .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
                .limit(recommend_dimension_size)
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java
@@ -50,7 +50,7 @@ public class SemanticService {
        return schemaService.getSemanticSchema();
    }

-    public ViewSchema getModelSchema(Long id) {
+    public ViewSchema getViewSchema(Long id) {
        return schemaService.getViewSchema(id);
    }

--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java
@@ -219,7 +219,7 @@ public class ConfigServiceImpl implements ConfigService {
        }
        BeanUtils.copyProperties(chatConfigResp, chatConfigRich);

-        ViewSchema viewSchema = semanticService.getModelSchema(modelId);
+        ViewSchema viewSchema = semanticService.getViewSchema(modelId);
        if (viewSchema == null) {
            return chatConfigRich;
        }
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
@@ -48,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService {
        if (Objects.isNull(modelId)) {
            return new RecommendResp();
        }
-        ViewSchema modelSchema = semanticService.getModelSchema(modelId);
+        ViewSchema modelSchema = semanticService.getViewSchema(modelId);
        if (Objects.isNull(modelSchema)) {
            return new RecommendResp();
        }
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java
@@ -1,17 +1,19 @@
 package com.tencent.supersonic.chat.server.service.impl;

-import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.headless.core.knowledge.DictWord;
-import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
 import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;

 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;


@Service
@@ -28,7 +30,6 @@ public class WordService {

        addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words);
        addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
-        addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words);
        addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
        addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);

@@ -36,6 +37,7 @@ public class WordService {
    }

    private void addWordsByType(DictWordType value, List<SchemaElement> metas, List<DictWord> natures) {
+        metas = distinct(metas);
        List<DictWord> natureList = WordBuilderFactory.get(value).getDictWords(metas);
        log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
        natures.addAll(natureList);
@@ -48,4 +50,13 @@ public class WordService {
    public void setPreDictWords(List<DictWord> preDictWords) {
        this.preDictWords = preDictWords;
    }
+
+    private List<SchemaElement> distinct(List<SchemaElement> metas) {
+        return metas.stream()
+                .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1))
+                .values()
+                .stream()
+                .collect(Collectors.toList());
+    }
+
 }