[improvement][chat] Fix the display of aliases or synonyms for terms (#1793)

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/DictWord.java

@@ -1,6 +1,9 @@
 package com.tencent.supersonic.headless.chat.knowledge;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.ToString;
 
 import java.util.Objects;
@@ -8,6 +11,9 @@ import java.util.Objects;
 /** * word nature */
 @Data
 @ToString
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
 public class DictWord {
 
     private String word;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/DataSetWithSemanticType.java

@@ -8,13 +8,13 @@ import java.io.Serializable;
 
 @Data
 @ToString
-public class ModelWithSemanticType implements Serializable {
+public class DataSetWithSemanticType implements Serializable {
 
-    private Long model;
+    private Long dataSetId;
     private SchemaElementType schemaElementType;
 
-    public ModelWithSemanticType(Long model, SchemaElementType schemaElementType) {
+    public DataSetWithSemanticType(Long dataSetId, SchemaElementType schemaElementType) {
-        this.model = model;
+        this.dataSetId = dataSetId;
         this.schemaElementType = schemaElementType;
     }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java

@@ -1,12 +1,10 @@
 package com.tencent.supersonic.headless.server.service.impl;
 
-import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
-import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
 import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
 import com.tencent.supersonic.headless.api.pojo.response.S2Term;
@@ -18,8 +16,8 @@ import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
 import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
 import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
+import com.tencent.supersonic.headless.chat.mapper.DataSetWithSemanticType;
 import com.tencent.supersonic.headless.chat.mapper.MatchText;
-import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
 import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
 import com.tencent.supersonic.headless.server.service.DataSetService;
 import com.tencent.supersonic.headless.server.service.RetrieveService;
@@ -31,6 +29,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -62,18 +61,18 @@ public class RetrieveServiceImpl implements RetrieveService {
 
     @Override
     public List<SearchResult> retrieve(QueryNLReq queryNLReq) {
-
         String queryText = queryNLReq.getQueryText();
-        // 1.get meta info
+
+        // 1. Get meta info
         SemanticSchema semanticSchemaDb =
                 schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
-        List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
+        Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
-        final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
         Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(
                 new ArrayList<>(dataSetIdToName.keySet()), User.getDefaultUser());
-        // 2.detect by segment
+
+        // 2. Detect by segment
         List<S2Term> originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds);
-        log.debug("hanlp parse result: {}", originals);
+        log.debug("originals terms: {}", originals);
         Set<Long> dataSetIds = queryNLReq.getDataSetIds();
 
         ChatQueryContext chatQueryContext = new ChatQueryContext();
@@ -82,47 +81,42 @@ public class RetrieveServiceImpl implements RetrieveService {
 
         Map<MatchText, List<HanlpMapResult>> regTextMap =
                 searchMatchStrategy.match(chatQueryContext, originals, dataSetIds);
+        regTextMap.values().forEach(HanlpHelper::transLetterOriginal);
 
-        regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
+        // 3. Get the most matching data
+        Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult = regTextMap
+                .entrySet().stream().filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
+                .max(Comparator.comparingInt(entry -> entry.getKey().getDetectSegment().length()));
 
-        // 3.get the most matching data
-        Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult =
-                regTextMap.entrySet().stream()
-                        .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
-                        .reduce((entry1, entry2) -> entry1.getKey().getDetectSegment()
-                                .length() >= entry2.getKey().getDetectSegment().length() ? entry1
-                                        : entry2);
-
-        // 4.optimize the results after the query
         if (!mostSimilarSearchResult.isPresent()) {
-            return Lists.newArrayList();
+            return Collections.emptyList();
         }
+
         Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry = mostSimilarSearchResult.get();
         log.debug("searchTextEntry:{},queryNLReq:{}", searchTextEntry, queryNLReq);
 
-        Set<SearchResult> searchResults = new LinkedHashSet();
-        DataSetInfoStat dataSetInfoStat = NatureHelper.getDataSetStat(originals);
 
+        DataSetInfoStat dataSetInfoStat = NatureHelper.getDataSetStat(originals);
         List<Long> possibleDataSets = getPossibleDataSets(queryNLReq, originals, dataSetIds);
 
-        // 5.1 priority dimension metric
+        // 5.1 Priority dimension metric
-        boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDataSets),
+        Set<SearchResult> searchResults = searchMetricAndDimension(new HashSet<>(possibleDataSets),
-                dataSetIdToName, searchTextEntry, searchResults);
+                dataSetIdToName, searchTextEntry);
+        boolean existMetricAndDimension = CollectionUtils.isNotEmpty(searchResults);
 
-        // 5.2 process based on dimension values
+        // 5.2 Process based on dimension values
         MatchText matchText = searchTextEntry.getKey();
         Map<String, String> natureToNameMap =
                 getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDataSets));
         log.debug("possibleDataSets:{},natureToNameMap:{}", possibleDataSets, natureToNameMap);
 
         for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
-
+            Set<SearchResult> results = searchDimensionValue(semanticSchemaDb,
-            Set<SearchResult> searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName,
                     dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension, matchText,
                     natureToNameMap, natureToNameEntry, queryNLReq.getQueryFilters());
-
+            searchResults.addAll(results);
-            searchResults.addAll(searchResultSet);
         }
+
         return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
     }
 
@@ -142,83 +136,89 @@ public class RetrieveServiceImpl implements RetrieveService {
         return possibleDataSets;
     }
 
-    private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
+    private Set<SearchResult> searchDimensionValue(SemanticSchema semanticSchemaDb,
-            Map<Long, String> modelToName, long metricModelCount, boolean existMetricAndDimension,
+            long metricModelCount, boolean existMetricAndDimension, MatchText matchText,
-            MatchText matchText, Map<String, String> natureToNameMap,
+            Map<String, String> natureToNameMap, Map.Entry<String, String> natureToNameEntry,
-            Map.Entry<String, String> natureToNameEntry, QueryFilters queryFilters) {
+            QueryFilters queryFilters) {
+        List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
+        Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
 
-        Set<SearchResult> searchResults = new LinkedHashSet();
+        Set<SearchResult> searchResults = new LinkedHashSet<>();
         String nature = natureToNameEntry.getKey();
         String wordName = natureToNameEntry.getValue();
 
-        Long modelId = NatureHelper.getDataSetId(nature);
+        Long dataSetId = NatureHelper.getDataSetId(nature);
         SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature);
 
+        // Skip if the schema element type is ENTITY
         if (SchemaElementType.ENTITY.equals(schemaElementType)) {
             return searchResults;
         }
-        // If there are no metric/dimension, complete the metric information
-        SearchResult searchResult = SearchResult.builder().modelId(modelId)
-                .modelName(modelToName.get(modelId)).recommend(matchText.getRegText() + wordName)
-                .schemaElementType(schemaElementType).subRecommend(wordName).build();
 
-        if (metricModelCount <= 0 && !existMetricAndDimension) {
+        // Create a base search result
+        SearchResult baseSearchResult = createBaseSearchResult(dataSetId, dataSetIdToName,
+                matchText, wordName, schemaElementType);
+
+        // If there are no metrics or dimensions, complete the metric information
+        if (shouldCompleteMetricInfo(metricModelCount, existMetricAndDimension)) {
             if (filterByQueryFilter(wordName, queryFilters)) {
                 return searchResults;
             }
-            searchResults.add(searchResult);
+            searchResults.add(baseSearchResult);
-            int metricSize = getMetricSize(natureToNameMap);
+
-            List<String> metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream()
+            int metricSize = calculateMetricSize(natureToNameMap);
-                    .limit(metricSize).collect(Collectors.toList());
+            List<String> metrics = getFilteredMetrics(metricsDb, dataSetId, metricSize);
 
             for (String metric : metrics) {
-                SearchResult result = SearchResult.builder().modelId(modelId)
+                SearchResult metricSearchResult = createMetricSearchResult(dataSetId,
-                        .modelName(modelToName.get(modelId))
+                        dataSetIdToName, matchText, wordName, metric);
-                        .recommend(matchText.getRegText() + wordName + DictWordType.SPACE + metric)
+                searchResults.add(metricSearchResult);
-                        .subRecommend(wordName + DictWordType.SPACE + metric).isComplete(false)
-                        .build();
-                searchResults.add(result);
             }
         } else {
-            searchResults.add(searchResult);
+            searchResults.add(baseSearchResult);
         }
+
         return searchResults;
     }
 
-    private int getMetricSize(Map<String, String> natureToNameMap) {
+    private SearchResult createBaseSearchResult(Long dataSetId, Map<Long, String> dataSetIdToName,
-        int metricSize = RESULT_SIZE / (natureToNameMap.entrySet().size());
+            MatchText matchText, String wordName, SchemaElementType schemaElementType) {
-        if (metricSize <= 1) {
+        return SearchResult.builder().modelId(dataSetId).modelName(dataSetIdToName.get(dataSetId))
-            metricSize = 1;
+                .recommend(matchText.getRegText() + wordName).schemaElementType(schemaElementType)
+                .subRecommend(wordName).build();
     }
-        return metricSize;
+
+    private boolean shouldCompleteMetricInfo(long metricModelCount,
+            boolean existMetricAndDimension) {
+        return metricModelCount <= 0 && !existMetricAndDimension;
+    }
+
+    private int calculateMetricSize(Map<String, String> natureToNameMap) {
+        int metricSize = RESULT_SIZE / natureToNameMap.size();
+        return Math.max(metricSize, 1);
+    }
+
+    private List<String> getFilteredMetrics(List<SchemaElement> metricsDb, Long modelId,
+            int metricSize) {
+        return metricsDb.stream()
+                .filter(mapDO -> Objects.nonNull(mapDO) && modelId.equals(mapDO.getDataSetId()))
+                .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
+                .map(SchemaElement::getName).limit(metricSize).collect(Collectors.toList());
+    }
+
+    private SearchResult createMetricSearchResult(Long modelId, Map<Long, String> modelToName,
+            MatchText matchText, String wordName, String metric) {
+        return SearchResult.builder().modelId(modelId).modelName(modelToName.get(modelId))
+                .recommend(matchText.getRegText() + wordName + DictWordType.SPACE + metric)
+                .subRecommend(wordName + DictWordType.SPACE + metric).isComplete(false).build();
     }
 
     private boolean filterByQueryFilter(String wordName, QueryFilters queryFilters) {
         if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) {
             return false;
         }
-        List<QueryFilter> filters = queryFilters.getFilters();
+        return queryFilters.getFilters().stream()
-        for (QueryFilter filter : filters) {
+                .noneMatch(filter -> wordName.equalsIgnoreCase(String.valueOf(filter.getValue())));
-            if (wordName.equalsIgnoreCase(String.valueOf(filter.getValue()))) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    protected List<String> filerMetricsByModel(List<SchemaElement> metricsDb, Long model,
-            int metricSize) {
-        if (CollectionUtils.isEmpty(metricsDb)) {
-            return Lists.newArrayList();
-        }
-        return metricsDb.stream()
-                .filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getDataSetId()))
-                .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
-                .flatMap(entry -> {
-                    List<String> result = new ArrayList<>();
-                    result.add(entry.getName());
-                    return result.stream();
-                }).limit(metricSize).collect(Collectors.toList());
     }
 
     /**
@@ -230,65 +230,71 @@ public class RetrieveServiceImpl implements RetrieveService {
     private Map<String, String> getNatureToNameMap(
             Map.Entry<MatchText, List<HanlpMapResult>> recommendTextListEntry,
             Set<Long> possibleModels) {
+
         List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
-        return recommendValues.stream()
+
-                .flatMap(entry -> entry.getNatures().stream().filter(nature -> {
+        return recommendValues.stream().flatMap(entry -> {
+            List<String> filteredNatures = entry.getNatures().stream()
+                    .filter(nature -> isNatureValid(nature, possibleModels))
+                    .collect(Collectors.toList());
+
+            return filteredNatures.stream()
+                    .map(nature -> DictWord.builder().word(entry.getName()).nature(nature).build());
+        }).sorted(Comparator.comparingInt(dictWord -> dictWord.getWord().length()))
+                .collect(Collectors.toMap(DictWord::getNature, DictWord::getWord,
+                        (value1, value2) -> value1, LinkedHashMap::new));
+    }
+
+    private boolean isNatureValid(String nature, Set<Long> possibleModels) {
         if (CollectionUtils.isEmpty(possibleModels)) {
             return true;
         }
         Long model = NatureHelper.getDataSetId(nature);
         return possibleModels.contains(model);
-                }).map(nature -> {
-                    DictWord posDO = new DictWord();
-                    posDO.setWord(entry.getName());
-                    posDO.setNature(nature);
-                    return posDO;
-                })).sorted(Comparator.comparingInt(a -> a.getWord().length()))
-                .collect(Collectors.toMap(DictWord::getNature, DictWord::getWord,
-                        (value1, value2) -> value1, LinkedHashMap::new));
     }
 
-    private boolean searchMetricAndDimension(Set<Long> possibleDataSets,
+    private Set<SearchResult> searchMetricAndDimension(Set<Long> possibleDataSets,
-            Map<Long, String> modelToName,
+            Map<Long, String> dataSetIdToName,
-            Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry,
+            Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry) {
-            Set<SearchResult> searchResults) {
+
-        boolean existMetric = false;
+        Set<SearchResult> searchResults = new LinkedHashSet<>();
         log.debug("searchMetricAndDimension searchTextEntry:{}", searchTextEntry);
+
         MatchText matchText = searchTextEntry.getKey();
         List<HanlpMapResult> hanlpMapResults = searchTextEntry.getValue();
 
         for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
-
+            List<DataSetWithSemanticType> dimensionMetricDataSetIds = hanlpMapResult.getNatures()
-            List<ModelWithSemanticType> dimensionMetricClassIds = hanlpMapResult.getNatures()
                     .stream()
-                    .map(nature -> new ModelWithSemanticType(NatureHelper.getDataSetId(nature),
+                    .map(nature -> new DataSetWithSemanticType(NatureHelper.getDataSetId(nature),
                             NatureHelper.convertToElementType(nature)))
                     .filter(entry -> matchCondition(entry, possibleDataSets))
                     .collect(Collectors.toList());
 
-            if (CollectionUtils.isEmpty(dimensionMetricClassIds)) {
+            if (CollectionUtils.isEmpty(dimensionMetricDataSetIds)) {
                 continue;
             }
-            for (ModelWithSemanticType modelWithSemanticType : dimensionMetricClassIds) {
+            for (DataSetWithSemanticType dataSetWithSemanticType : dimensionMetricDataSetIds) {
-                existMetric = true;
+                Long dataSetId = dataSetWithSemanticType.getDataSetId();
-                Long modelId = modelWithSemanticType.getModel();
+                SchemaElementType schemaElementType =
-                SchemaElementType schemaElementType = modelWithSemanticType.getSchemaElementType();
+                        dataSetWithSemanticType.getSchemaElementType();
+                String modelName = dataSetIdToName.get(dataSetId);
+                String recommendText = matchText.getRegText() + hanlpMapResult.getName();
+                String subRecommendText = hanlpMapResult.getName();
+
                 SearchResult searchResult =
-                        SearchResult.builder().modelId(modelId).modelName(modelToName.get(modelId))
+                        SearchResult.builder().modelId(dataSetId).modelName(modelName)
-                                .recommend(matchText.getRegText() + hanlpMapResult.getName())
+                                .recommend(recommendText).subRecommend(subRecommendText)
-                                .subRecommend(hanlpMapResult.getName())
                                 .schemaElementType(schemaElementType).build();
-                // visibility to filter metrics
+
                 searchResults.add(searchResult);
             }
-            log.debug("parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}",
-                    hanlpMapResult, dimensionMetricClassIds, possibleDataSets);
         }
         log.info("searchMetricAndDimension searchResults:{}", searchResults);
-        return existMetric;
+        return searchResults;
     }
 
-    private boolean matchCondition(ModelWithSemanticType entry, Set<Long> possibleDataSets) {
+    private boolean matchCondition(DataSetWithSemanticType entry, Set<Long> possibleDataSets) {
         if (!(SchemaElementType.METRIC.equals(entry.getSchemaElementType())
                 || SchemaElementType.DIMENSION.equals(entry.getSchemaElementType()))) {
             return false;
@@ -297,6 +303,6 @@ public class RetrieveServiceImpl implements RetrieveService {
         if (CollectionUtils.isEmpty(possibleDataSets)) {
             return true;
         }
-        return possibleDataSets.contains(entry.getModel());
+        return possibleDataSets.contains(entry.getDataSetId());
     }
 }

launchers/standalone/src/main/java/com/tencent/supersonic/config/SwaggerConfiguration.java

@@ -5,6 +5,10 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+
+import java.util.Collections;
+import java.util.List;
+
 import springfox.documentation.builders.ApiInfoBuilder;
 import springfox.documentation.builders.PathSelectors;
 import springfox.documentation.builders.RequestHandlerSelectors;
@@ -18,9 +22,6 @@ import springfox.documentation.spi.service.contexts.SecurityContext;
 import springfox.documentation.spring.web.plugins.Docket;
 import springfox.documentation.swagger2.annotations.EnableSwagger2;
 
-import java.util.Collections;
-import java.util.List;
-
 @Configuration
 @EnableSwagger2
 @EnableOpenApi