(improvement)(chat) queryDimensionValue query top10 in queue (#277)

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java

@@ -1,16 +1,21 @@
 package com.tencent.supersonic.chat.api.pojo.request;
 
+import javax.validation.constraints.NotNull;
 import lombok.Data;
 
 @Data
 public class DimensionValueReq {
+
     private Integer agentId;
 
+    @NotNull
     private Long elementID;
 
+    @NotNull
     private Long modelId;
 
     private String bizName;
 
+    @NotNull
     private Object value;
 }

chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java

@@ -94,8 +94,4 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
         return semanticSchema.getMetrics(modelId);
     }
 
-    protected List<SchemaElement> getDimensionElements(Long modelId) {
-        SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
-        return semanticSchema.getDimensions(modelId);
-    }
 }

chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java

@@ -7,6 +7,7 @@ import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.knowledge.service.SchemaService;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -32,9 +33,20 @@ public class GroupByCorrector extends BaseSemanticCorrector {
         //add dimension group by
         String sql = semanticCorrectInfo.getSql();
         SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
+        //add alias field name
         Set<String> dimensions = semanticSchema.getDimensions(modelId).stream()
-                .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
+                .flatMap(
+                        schemaElement -> {
+                            Set<String> elements = new HashSet<>();
+                            elements.add(schemaElement.getName());
+                            if (!CollectionUtils.isEmpty(schemaElement.getAlias())) {
+                                elements.addAll(schemaElement.getAlias());
+                            }
+                            return elements.stream();
+                        }
+                ).collect(Collectors.toSet());
         dimensions.add(DateUtils.DATE_FIELD);
+
         List<String> selectFields = SqlParserSelectHelper.getSelectFields(sql);
 
         if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) {

chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java

@@ -11,6 +11,7 @@ import com.tencent.supersonic.chat.service.QueryService;
 import com.tencent.supersonic.chat.service.SearchService;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import javax.validation.Valid;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -78,7 +79,7 @@ public class ChatQueryController {
     }
 
     @PostMapping("queryDimensionValue")
-    public Object queryDimensionValue(@RequestBody DimensionValueReq dimensionValueReq,
+    public Object queryDimensionValue(@RequestBody @Valid DimensionValueReq dimensionValueReq,
                                       HttpServletRequest request, HttpServletResponse response)
             throws Exception {
         return queryService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response));

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.chat.service.impl;
 
 
+import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.component.SchemaMapper;
 import com.tencent.supersonic.chat.api.component.SemanticParser;
@@ -37,6 +38,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
 import com.tencent.supersonic.chat.utils.SolvedQueryManager;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.QueryColumn;
+import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
@@ -46,6 +48,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.knowledge.dictionary.MapResult;
+import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary;
 import com.tencent.supersonic.knowledge.service.SearchService;
 import com.tencent.supersonic.knowledge.utils.NatureHelper;
 import com.tencent.supersonic.knowledge.utils.HanlpHelper;
@@ -59,6 +62,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -590,8 +594,14 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
-        if (Objects.isNull(dimensionValueReq.getValue())) {
+        if (StringUtils.isBlank(dimensionValueReq.getValue().toString())) {
-            dimensionValueReq.setValue("");
+            String nature =
+                    dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID();
+            PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
+            if (CollectionUtils.isEmpty(terms)) {
+                return null;
+            }
+            return terms.stream().map(term -> term.getWord()).collect(Collectors.toSet());
         }
         return queryHanlpDimensionValue(dimensionValueReq, user);
     }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java

@@ -11,12 +11,12 @@ import com.hankcs.hanlp.corpus.tag.Nature;
 import com.hankcs.hanlp.dictionary.CoreDictionary;
 import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
 import com.hankcs.hanlp.dictionary.other.CharTable;
+import com.hankcs.hanlp.seg.common.Term;
 import com.hankcs.hanlp.utility.LexiconUtility;
 import com.hankcs.hanlp.utility.Predefine;
 import com.hankcs.hanlp.utility.TextUtility;
 import com.tencent.supersonic.knowledge.service.SearchService;
 import com.tencent.supersonic.knowledge.utils.HanlpHelper;
-
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
@@ -25,15 +25,21 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.PriorityQueue;
 import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 public class MultiCustomDictionary extends DynamicCustomDictionary {
 
+    public static int MAX_SIZE = 10;
     public static Boolean removeDuplicates = true;
+    public static ConcurrentHashMap<String, PriorityQueue<Term>> NATURE_TO_VALUES = new ConcurrentHashMap<>();
     private static boolean addToSuggesterTrie = true;
 
     public MultiCustomDictionary() {
@@ -107,17 +113,26 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
                     }
                 }
                 attribute.original = original;
+
                 if (removeDuplicates && map.containsKey(word)) {
                     attribute = DictionaryAttributeUtil.getAttribute(map.get(word), attribute);
+                }
                 map.put(word, attribute);
                 if (addToSuggeterTrie) {
                     SearchService.put(word, attribute);
                 }
-
+                for (int i = 0; i < attribute.nature.length; i++) {
-                } else {
+                    Nature nature = attribute.nature[i];
-                    map.put(word, attribute);
+                    PriorityQueue<Term> priorityQueue = NATURE_TO_VALUES.get(nature.toString());
-                    if (addToSuggeterTrie) {
+                    if (Objects.isNull(priorityQueue)) {
-                        SearchService.put(word, attribute);
+                        priorityQueue = new PriorityQueue<>(MAX_SIZE,
+                                Comparator.comparingInt(Term::getFrequency).reversed());
+                        NATURE_TO_VALUES.put(nature.toString(), priorityQueue);
+                    }
+                    Term term = new Term(word, nature);
+                    term.setFrequency(attribute.frequency[i]);
+                    if (!priorityQueue.contains(term) && priorityQueue.size() < MAX_SIZE) {
+                        priorityQueue.add(term);
                     }
                 }
             }