diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java index 5ac322b68..0f10d7d39 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java @@ -10,10 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.knowledge.dictionary.MapResult; -import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder; -import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import com.tencent.supersonic.knowledge.utils.NatureHelper; import java.util.ArrayList; @@ -99,8 +96,7 @@ public class HanlpDictMapper implements SchemaMapper { SemanticService schemaService = ContextUtils.getBean(SemanticService.class); ModelSchema modelSchema = schemaService.getModelSchema(modelId); - BaseWordBuilder baseWordBuilder = WordBuilderFactory.get(DictWordType.getNatureType(nature)); - Long elementID = baseWordBuilder.getElementID(nature); + Long elementID = NatureHelper.getElementID(nature); Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature); SchemaElement elementDb = modelSchema.getElement(elementType, elementID); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index b913b5a61..84417f071 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -1,10 +1,8 @@ package com.tencent.supersonic.chat.service.impl; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SchemaMapper; -import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; @@ -27,8 +25,8 @@ import com.tencent.supersonic.chat.persistence.dataobject.CostType; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QuerySelector; -import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; +import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.service.ChatService; @@ -38,46 +36,43 @@ import com.tencent.supersonic.chat.service.StatisticsService; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.SolvedQueryManager; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.dictionary.MapResult; import com.tencent.supersonic.knowledge.service.SearchService; +import com.tencent.supersonic.knowledge.utils.NatureHelper; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; -import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; - -import java.util.Map; -import java.util.HashMap; -import java.util.Objects; -import java.util.Set; -import java.util.HashSet; -import java.util.List; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.operators.relational.EqualsTo; -import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; -import net.sf.jsqlparser.expression.operators.relational.GreaterThan; -import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; -import net.sf.jsqlparser.expression.operators.relational.MinorThan; -import net.sf.jsqlparser.expression.operators.relational.InExpression; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; +import net.sf.jsqlparser.expression.operators.relational.EqualsTo; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.GreaterThan; +import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.MinorThan; +import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.schema.Column; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.commons.collections.CollectionUtils; @@ -181,7 +176,7 @@ public class QueryServiceImpl implements QueryService { } private List getTop5CandidateParseInfo(List selectedParses, - List candidateParses) { + List candidateParses) { if (CollectionUtils.isEmpty(selectedParses) || CollectionUtils.isEmpty(candidateParses)) { return candidateParses; } @@ -398,10 +393,10 @@ public class QueryServiceImpl implements QueryService { } private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo, - Map> filedNameToValueMap, - List filterExpressionList, - List addConditions, - Set removeFieldNames) { + Map> filedNameToValueMap, + List filterExpressionList, + List addConditions, + Set removeFieldNames) { if (Objects.isNull(queryData.getDateInfo())) { return; } @@ -460,8 +455,8 @@ public class QueryServiceImpl implements QueryService { } public void addTimeCondition(String date, - T comparisonExpression, - List addConditions) { + T comparisonExpression, + List addConditions) { Column column = new Column(DateUtils.DATE_FIELD); StringValue stringValue = new StringValue(date); comparisonExpression.setLeftExpression(column); @@ -470,11 +465,11 @@ public class QueryServiceImpl implements QueryService { } private void updateFilters(Map> filedNameToValueMap, - List filterExpressionList, - Set metricFilters, - Set contextMetricFilters, - List addConditions, - Set removeFieldNames) { + List filterExpressionList, + Set metricFilters, + Set contextMetricFilters, + List addConditions, + Set removeFieldNames) { if (CollectionUtils.isEmpty(metricFilters)) { return; } @@ -521,9 +516,9 @@ public class QueryServiceImpl implements QueryService { } public void addWhereInCondition(QueryFilter dslQueryFilter, - InExpression inExpression, - Set contextMetricFilters, - List addConditions) { + InExpression inExpression, + Set contextMetricFilters, + List addConditions) { Column column = new Column(dslQueryFilter.getName()); ExpressionList expressionList = new ExpressionList(); List expressions = new ArrayList<>(); @@ -549,9 +544,9 @@ public class QueryServiceImpl implements QueryService { } public void addWhereCondition(QueryFilter dslQueryFilter, - T comparisonExpression, - Set contextMetricFilters, - List addConditions) { + T comparisonExpression, + Set contextMetricFilters, + List addConditions) { String columnName = dslQueryFilter.getName(); if (StringUtils.isNotBlank(dslQueryFilter.getFunction())) { columnName = dslQueryFilter.getFunction() + "(" + dslQueryFilter.getName() + ")"; @@ -595,26 +590,10 @@ public class QueryServiceImpl implements QueryService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { - QueryStructReq queryStructReq = new QueryStructReq(); - - DateConf dateConf = new DateConf(); - dateConf.setDateMode(DateConf.DateMode.RECENT); - dateConf.setUnit(1); - dateConf.setPeriod("DAY"); - queryStructReq.setDateInfo(dateConf); - queryStructReq.setLimit(20L); - queryStructReq.setModelId(dimensionValueReq.getModelId()); - queryStructReq.setNativeQuery(false); - List groups = new ArrayList<>(); - groups.add(dimensionValueReq.getBizName()); - queryStructReq.setGroups(groups); - if ((!Objects.isNull(dimensionValueReq.getValue())) - && StringUtils.isNotBlank(dimensionValueReq.getValue().toString())) { - return queryHanlpDimensionValue(dimensionValueReq, user); + if (Objects.isNull(dimensionValueReq.getValue())) { + dimensionValueReq.setValue(""); } - SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); - QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user); - return queryResultWithSchemaResp; + return queryHanlpDimensionValue(dimensionValueReq, user); } public Object queryHanlpDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { @@ -624,11 +603,10 @@ public class QueryServiceImpl implements QueryService { List mapResultList = SearchService.prefixSearch(dimensionValueReq.getValue().toString(), 2000, dimensionValueReq.getAgentId(), detectModelIds); HanlpHelper.transLetterOriginal(mapResultList); - log.info("mapResultList:{}", mapResultList); mapResultList = mapResultList.stream().filter(o -> { for (String nature : o.getNatures()) { - String[] natureArray = nature.split("_"); - if (natureArray[2].equals(dimensionValueReq.getElementID().toString())) { + Long elementID = NatureHelper.getElementID(nature); + if (dimensionValueReq.getElementID().equals(elementID)) { return true; } } diff --git a/chat/knowledge/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java b/chat/knowledge/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java index 0996363ed..f4c3dd507 100644 --- a/chat/knowledge/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java +++ b/chat/knowledge/src/main/java/com/hankcs/hanlp/collection/trie/bintrie/BaseNode.java @@ -12,6 +12,7 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import org.slf4j.Logger; @@ -322,8 +323,10 @@ public abstract class BaseNode implements Comparable { } String prefix = root.prefix + root.c; for (BaseNode node : root.child) { - node.prefix = prefix; - queue.add(node); + if (Objects.nonNull(node)) { + node.prefix = prefix; + queue.add(node); + } } } } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java index 219c4d072..768f02ada 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/BaseWordBuilder.java @@ -1,11 +1,9 @@ package com.tencent.supersonic.knowledge.dictionary.builder; -import java.util.ArrayList; -import java.util.List; - import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import java.util.ArrayList; +import java.util.List; import lombok.extern.slf4j.Slf4j; /** @@ -37,13 +35,4 @@ public abstract class BaseWordBuilder { } protected abstract List doGet(String word, SchemaElement schemaElement); - - public Long getElementID(String nature) { - String[] split = nature.split(DictWordType.NATURE_SPILT); - if (split.length >= 3) { - return Long.valueOf(split[2]); - } - return 0L; - } - } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java index 75dd6f66e..1f854d12b 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SearchService.java @@ -86,7 +86,11 @@ public class SearchService { BinTrie> binTrie, Integer agentId, Set detectModelIds) { key = key.toLowerCase(); Set>> entrySet = new TreeSet>>(); - StringBuilder sb = new StringBuilder(key.substring(0, key.length() - 1)); + + StringBuilder sb = new StringBuilder(); + if (StringUtils.isNotBlank(key)) { + sb = new StringBuilder(key.substring(0, key.length() - 1)); + } BaseNode branch = binTrie; char[] chars = key.toCharArray(); for (char aChar : chars) { diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java index c3e945e15..02f946b39 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/NatureHelper.java @@ -165,4 +165,13 @@ public class NatureHelper { return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxModelTypeSize) .map(entry -> entry.getKey()).collect(Collectors.toList()); } + + public static Long getElementID(String nature) { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length >= 3) { + return Long.valueOf(split[2]); + } + return 0L; + } + }