(improvement)(chat)Dimension value queries are uniformly conducted through HanLP dictionary queries (#271)

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java

@@ -10,10 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.service.SemanticService;
 import com.tencent.supersonic.common.util.ContextUtils;
-import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.knowledge.dictionary.MapResult;
-import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
-import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory;
 import com.tencent.supersonic.knowledge.utils.HanlpHelper;
 import com.tencent.supersonic.knowledge.utils.NatureHelper;
 import java.util.ArrayList;
@@ -99,8 +96,7 @@ public class HanlpDictMapper implements SchemaMapper {
                 SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
                 ModelSchema modelSchema = schemaService.getModelSchema(modelId);
 
-                BaseWordBuilder baseWordBuilder = WordBuilderFactory.get(DictWordType.getNatureType(nature));
+                Long elementID = NatureHelper.getElementID(nature);
-                Long elementID = baseWordBuilder.getElementID(nature);
                 Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
 
                 SchemaElement elementDb = modelSchema.getElement(elementType, elementID);

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -1,10 +1,8 @@
 package com.tencent.supersonic.chat.service.impl;
 
 
-
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.component.SchemaMapper;
-import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
 import com.tencent.supersonic.chat.api.component.SemanticParser;
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -27,8 +25,8 @@ import com.tencent.supersonic.chat.persistence.dataobject.CostType;
 import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
 import com.tencent.supersonic.chat.query.QueryManager;
 import com.tencent.supersonic.chat.query.QuerySelector;
-import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
 import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
+import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
 import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
 import com.tencent.supersonic.chat.responder.parse.ParseResponder;
 import com.tencent.supersonic.chat.service.ChatService;
@@ -38,46 +36,43 @@ import com.tencent.supersonic.chat.service.StatisticsService;
 import com.tencent.supersonic.chat.utils.ComponentFactory;
 import com.tencent.supersonic.chat.utils.SolvedQueryManager;
 import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
 import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
-import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.knowledge.dictionary.MapResult;
 import com.tencent.supersonic.knowledge.service.SearchService;
+import com.tencent.supersonic.knowledge.utils.NatureHelper;
 import com.tencent.supersonic.knowledge.utils.HanlpHelper;
 import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Objects;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.List;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
-
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.LongValue;
 import net.sf.jsqlparser.expression.StringValue;
-import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
-import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
-import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
-import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
-import net.sf.jsqlparser.expression.operators.relational.MinorThan;
-import net.sf.jsqlparser.expression.operators.relational.InExpression;
 import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
+import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
 import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
+import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
+import net.sf.jsqlparser.expression.operators.relational.InExpression;
+import net.sf.jsqlparser.expression.operators.relational.MinorThan;
+import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
 import net.sf.jsqlparser.schema.Column;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.commons.collections.CollectionUtils;
@@ -595,26 +590,10 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
-        QueryStructReq queryStructReq = new QueryStructReq();
+        if (Objects.isNull(dimensionValueReq.getValue())) {
-
+            dimensionValueReq.setValue("");
-        DateConf dateConf = new DateConf();
-        dateConf.setDateMode(DateConf.DateMode.RECENT);
-        dateConf.setUnit(1);
-        dateConf.setPeriod("DAY");
-        queryStructReq.setDateInfo(dateConf);
-        queryStructReq.setLimit(20L);
-        queryStructReq.setModelId(dimensionValueReq.getModelId());
-        queryStructReq.setNativeQuery(false);
-        List<String> groups = new ArrayList<>();
-        groups.add(dimensionValueReq.getBizName());
-        queryStructReq.setGroups(groups);
-        if ((!Objects.isNull(dimensionValueReq.getValue()))
-                && StringUtils.isNotBlank(dimensionValueReq.getValue().toString())) {
-            return queryHanlpDimensionValue(dimensionValueReq, user);
         }
-        SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
+        return queryHanlpDimensionValue(dimensionValueReq, user);
-        QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
-        return queryResultWithSchemaResp;
     }
 
     public Object queryHanlpDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
@@ -624,11 +603,10 @@ public class QueryServiceImpl implements QueryService {
         List<MapResult> mapResultList = SearchService.prefixSearch(dimensionValueReq.getValue().toString(),
                 2000, dimensionValueReq.getAgentId(), detectModelIds);
         HanlpHelper.transLetterOriginal(mapResultList);
-        log.info("mapResultList:{}", mapResultList);
         mapResultList = mapResultList.stream().filter(o -> {
             for (String nature : o.getNatures()) {
-                String[] natureArray = nature.split("_");
+                Long elementID = NatureHelper.getElementID(nature);
-                if (natureArray[2].equals(dimensionValueReq.getElementID().toString())) {
+                if (dimensionValueReq.getElementID().equals(elementID)) {
                     return true;
                 }
             }

chat/knowledge/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java

@@ -12,6 +12,7 @@ import java.util.ArrayDeque;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.Set;
 import org.slf4j.Logger;
@@ -322,10 +323,12 @@ public abstract class BaseNode<V> implements Comparable<BaseNode> {
             }
             String prefix = root.prefix + root.c;
             for (BaseNode node : root.child) {
+                if (Objects.nonNull(node)) {
                     node.prefix = prefix;
                     queue.add(node);
                 }
             }
         }
+    }
 
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java

@@ -1,11 +1,9 @@
 package com.tencent.supersonic.knowledge.dictionary.builder;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.knowledge.dictionary.DictWord;
-import com.tencent.supersonic.common.pojo.enums.DictWordType;
+import java.util.ArrayList;
+import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -37,13 +35,4 @@ public abstract class BaseWordBuilder {
     }
 
     protected abstract List<DictWord> doGet(String word, SchemaElement schemaElement);
-
-    public Long getElementID(String nature) {
-        String[] split = nature.split(DictWordType.NATURE_SPILT);
-        if (split.length >= 3) {
-            return Long.valueOf(split[2]);
-        }
-        return 0L;
-    }
-
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java

@@ -86,7 +86,11 @@ public class SearchService {
             BinTrie<List<String>> binTrie, Integer agentId, Set<Long> detectModelIds) {
         key = key.toLowerCase();
         Set<Map.Entry<String, List<String>>> entrySet = new TreeSet<Map.Entry<String, List<String>>>();
-        StringBuilder sb = new StringBuilder(key.substring(0, key.length() - 1));
+
+        StringBuilder sb = new StringBuilder();
+        if (StringUtils.isNotBlank(key)) {
+            sb = new StringBuilder(key.substring(0, key.length() - 1));
+        }
         BaseNode branch = binTrie;
         char[] chars = key.toCharArray();
         for (char aChar : chars) {

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java

@@ -165,4 +165,13 @@ public class NatureHelper {
         return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxModelTypeSize)
                 .map(entry -> entry.getKey()).collect(Collectors.toList());
     }
+
+    public static Long getElementID(String nature) {
+        String[] split = nature.split(DictWordType.NATURE_SPILT);
+        if (split.length >= 3) {
+            return Long.valueOf(split[2]);
+        }
+        return 0L;
+    }
+
 }