diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java index d969100b6..6f0871954 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java @@ -1,16 +1,21 @@ package com.tencent.supersonic.chat.api.pojo.request; +import javax.validation.constraints.NotNull; import lombok.Data; @Data public class DimensionValueReq { + private Integer agentId; + @NotNull private Long elementID; + @NotNull private Long modelId; private String bizName; + @NotNull private Object value; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java index ded941187..75d429dd6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java @@ -94,8 +94,4 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { return semanticSchema.getMetrics(modelId); } - protected List getDimensionElements(Long modelId) { - SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); - return semanticSchema.getDimensions(modelId); - } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java index fd312c3b4..40f1890f3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -32,9 +33,20 @@ public class GroupByCorrector extends BaseSemanticCorrector { //add dimension group by String sql = semanticCorrectInfo.getSql(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); + //add alias field name Set dimensions = semanticSchema.getDimensions(modelId).stream() - .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); + .flatMap( + schemaElement -> { + Set elements = new HashSet<>(); + elements.add(schemaElement.getName()); + if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { + elements.addAll(schemaElement.getAlias()); + } + return elements.stream(); + } + ).collect(Collectors.toSet()); dimensions.add(DateUtils.DATE_FIELD); + List selectFields = SqlParserSelectHelper.getSelectFields(sql); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java index a448b07b4..a0bff539f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.chat.service.QueryService; import com.tencent.supersonic.chat.service.SearchService; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.PostMapping; @@ -78,7 +79,7 @@ public class ChatQueryController { } @PostMapping("queryDimensionValue") - public Object queryDimensionValue(@RequestBody DimensionValueReq dimensionValueReq, + public Object queryDimensionValue(@RequestBody @Valid DimensionValueReq dimensionValueReq, HttpServletRequest request, HttpServletResponse response) throws Exception { return queryService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response)); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 84417f071..c9c4eab19 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.service.impl; +import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SchemaMapper; import com.tencent.supersonic.chat.api.component.SemanticParser; @@ -37,6 +38,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.SolvedQueryManager; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; @@ -46,6 +48,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.dictionary.MapResult; +import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary; import com.tencent.supersonic.knowledge.service.SearchService; import com.tencent.supersonic.knowledge.utils.NatureHelper; import com.tencent.supersonic.knowledge.utils.HanlpHelper; @@ -59,6 +62,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -590,8 +594,14 @@ public class QueryServiceImpl implements QueryService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { - if (Objects.isNull(dimensionValueReq.getValue())) { - dimensionValueReq.setValue(""); + if (StringUtils.isBlank(dimensionValueReq.getValue().toString())) { + String nature = + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID(); + PriorityQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); + if (CollectionUtils.isEmpty(terms)) { + return null; + } + return terms.stream().map(term -> term.getWord()).collect(Collectors.toSet()); } return queryHanlpDimensionValue(dimensionValueReq, user); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java index 67f041230..015775c29 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java @@ -11,12 +11,12 @@ import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.dictionary.other.CharTable; +import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.utility.LexiconUtility; import com.hankcs.hanlp.utility.Predefine; import com.hankcs.hanlp.utility.TextUtility; import com.tencent.supersonic.knowledge.service.SearchService; import com.tencent.supersonic.knowledge.utils.HanlpHelper; - import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.DataOutputStream; @@ -25,15 +25,21 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; +import java.util.Comparator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.PriorityQueue; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; public class MultiCustomDictionary extends DynamicCustomDictionary { + public static int MAX_SIZE = 10; public static Boolean removeDuplicates = true; + public static ConcurrentHashMap> NATURE_TO_VALUES = new ConcurrentHashMap<>(); private static boolean addToSuggesterTrie = true; public MultiCustomDictionary() { @@ -107,17 +113,26 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { } } attribute.original = original; + if (removeDuplicates && map.containsKey(word)) { attribute = DictionaryAttributeUtil.getAttribute(map.get(word), attribute); - map.put(word, attribute); - if (addToSuggeterTrie) { - SearchService.put(word, attribute); + } + map.put(word, attribute); + if (addToSuggeterTrie) { + SearchService.put(word, attribute); + } + for (int i = 0; i < attribute.nature.length; i++) { + Nature nature = attribute.nature[i]; + PriorityQueue priorityQueue = NATURE_TO_VALUES.get(nature.toString()); + if (Objects.isNull(priorityQueue)) { + priorityQueue = new PriorityQueue<>(MAX_SIZE, + Comparator.comparingInt(Term::getFrequency).reversed()); + NATURE_TO_VALUES.put(nature.toString(), priorityQueue); } - - } else { - map.put(word, attribute); - if (addToSuggeterTrie) { - SearchService.put(word, attribute); + Term term = new Term(word, nature); + term.setFrequency(attribute.frequency[i]); + if (!priorityQueue.contains(term) && priorityQueue.size() < MAX_SIZE) { + priorityQueue.add(term); } } }