From b6f0df40a95300d5f6588bebe218a7cd4be52bb6 Mon Sep 17 00:00:00 2001 From: lexluo Date: Thu, 15 Jun 2023 18:15:44 +0800 Subject: [PATCH] [improvement](chat) support query/search filter by web domainId and mapping add frequency/detectWord in mapping and metric dimensions orders filter duplicates --- .../supersonic/chat/api/pojo/Filter.java | 20 ++ .../chat/api/pojo/SchemaElementMatch.java | 14 +- .../chat/api/pojo/SchemaMapInfo.java | 4 + .../chat/api/pojo/SemanticParseInfo.java | 12 +- .../chat/api/service/SchemaMapper.java | 2 +- .../chat/application/ChatServiceImpl.java | 11 + .../chat/application/DomainEntityService.java | 12 +- .../chat/application/QueryServiceImpl.java | 5 + .../chat/application/SearchServiceImpl.java | 72 ++-- .../knowledge/ApplicationStartedInit.java | 9 +- .../application/knowledge/NatureHelper.java | 3 +- .../knowledge/WordNatureService.java | 35 +- .../application/mapper/HanlpSchemaMapper.java | 24 +- .../application/mapper/MatchStrategy.java | 12 +- .../mapper/QueryMatchStrategy.java | 36 +- .../mapper/SearchMatchStrategy.java | 24 +- .../parser/AggregateSemanticParser.java | 17 +- .../parser/DefaultMetricSemanticParser.java | 60 ++-- .../parser/DomainSemanticParser.java | 32 +- .../application/parser/ListFilterParser.java | 25 +- .../parser/resolver/BaseDomainResolver.java | 131 +++++++ .../resolver/HeuristicDomainResolver.java | 112 +----- .../resolver/RegexAggregateTypeResolver.java | 2 +- .../application/query/EntityListFilter.java | 2 + .../chat/application/query/MetricCompare.java | 67 +++- .../chat/domain/pojo/chat/QueryData.java | 16 +- .../repository/ChatQueryRepository.java | 4 + .../chat/domain/service/ChatService.java | 5 + .../chat/domain/utils/ContextHelper.java | 7 +- .../domain/utils/SchemaInfoConverter.java | 15 +- .../repository/ChatQueryRepositoryImpl.java | 22 ++ .../mapper/HanlpSchemaMapperTest.java | 10 +- .../parser/TimeSemanticParserTest.java | 8 +- .../context/SemanticParseObjectHelper.java | 8 +- .../hankcs/hanlp/seg/WordBasedSegment.java | 337 ++++++++++++++++++ .../application/online/BaseWordNature.java | 8 + .../online/DimensionWordNature.java | 6 +- .../application/online/DomainWordNature.java | 9 +- .../application/online/MetricWordNature.java | 6 +- .../nlp/DictionaryAttributeUtil.java | 3 + .../infrastructure/nlp/HanlpHelper.java | 17 + .../nlp/MultiCustomDictionary.java | 10 + .../infrastructure/nlp/Suggester.java | 4 +- .../supersonic/common/nlp/MapResult.java | 5 +- .../tencent/supersonic/common/pojo/Order.java | 19 + .../supersonic/common/pojo/SchemaItem.java | 23 ++ .../supersonic/common/util/RecordInfo.java | 19 + .../main/resources/META-INF/spring.factories | 3 + pom.xml | 25 ++ .../core/application/DatabaseServiceImpl.java | 1 + .../repository/DatasourceRepository.java | 2 +- .../domain/repository/MetricRepository.java | 2 +- .../repository/DatasourceRepositoryImpl.java | 4 +- .../repository/DateInfoRepositoryImpl.java | 25 -- .../SemanticSchemaManagerImpl.java | 2 +- .../{ => parser}/SemanticSqlServiceImpl.java | 2 +- .../parser/SemanticParserServiceTest.java | 2 +- 57 files changed, 1040 insertions(+), 332 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/BaseDomainResolver.java create mode 100644 chat/knowledge/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/{ => parser}/SemanticSchemaManagerImpl.java (99%) rename semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/{ => parser}/SemanticSqlServiceImpl.java (95%) diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java index 3e6bb6621..5b5d7a8aa 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; +import java.util.Objects; import lombok.Data; import lombok.ToString; @@ -17,4 +18,23 @@ public class Filter { private Object value; private Long elementID; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Filter filter = (Filter) o; + return Objects.equals(bizName, filter.bizName) && Objects.equals(name, filter.name) + && operator == filter.operator && Objects.equals(value, filter.value) && Objects.equals( + elementID, filter.elementID); + } + + @Override + public int hashCode() { + return Objects.hash(bizName, name, operator, value, elementID); + } } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java index 568ecd212..fd5f6cdf0 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java @@ -8,17 +8,17 @@ import lombok.ToString; public class SchemaElementMatch { SchemaElementType elementType; + int elementID; + double similarity; + + String detectWord; + String word; + Long frequency; + public SchemaElementMatch() { } - - public SchemaElementMatch(SchemaElementType schemaElementType, int elementID, double similarity, String word) { - this.elementID = elementID; - this.elementType = schemaElementType; - this.similarity = similarity; - this.word = word; - } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java index bff25261b..d13c840d6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java @@ -25,4 +25,8 @@ public class SchemaMapInfo { domainElementMatches.put(domain, elementMatches); } + public void setDomainElementMatches( + Map> domainElementMatches) { + this.domainElementMatches = domainElementMatches; + } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 5b9dbe201..c7bd543e0 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -6,7 +6,9 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.SchemaItem; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import lombok.Data; @Data @@ -17,11 +19,11 @@ public class SemanticParseInfo { Long domainId = 0L; String domainName; Long entity = 0L; - List metrics = new ArrayList<>(); - List dimensions = new ArrayList<>(); - List dimensionFilters = new ArrayList<>(); - List metricFilters = new ArrayList<>(); - private List orders = new ArrayList<>(); + Set metrics = new LinkedHashSet(); + Set dimensions = new LinkedHashSet(); + Set dimensionFilters = new LinkedHashSet(); + Set metricFilters = new LinkedHashSet(); + private Set orders = new LinkedHashSet(); private DateConf dateInfo; private Long limit; private Boolean nativeQuery = false; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/service/SchemaMapper.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/service/SchemaMapper.java index 9074ac916..7c89eca4f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/service/SchemaMapper.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/service/SchemaMapper.java @@ -11,5 +11,5 @@ import com.tencent.supersonic.chat.api.request.QueryContextReq; */ public interface SchemaMapper { - void map(QueryContextReq searchCtx); + void map(QueryContextReq queryContext); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/ChatServiceImpl.java index 689cbdefb..5663f3f9a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/ChatServiceImpl.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.response.QueryResultResp; import com.tencent.supersonic.chat.domain.dataobject.ChatDO; +import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.domain.dataobject.QueryDO; import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO; import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq; @@ -130,4 +131,14 @@ public class ChatServiceImpl implements ChatService { chatQueryRepository.createChatQuery(queryResponse, queryContext, chatCtx); } + @Override + public ChatQueryDO getLastQuery(long chatId) { + return chatQueryRepository.getLastChatQuery(chatId); + } + + @Override + public int updateQuery(ChatQueryDO chatQueryDO) { + return chatQueryRepository.updateChatQuery(chatQueryDO); + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java index 65ff321a3..052b918c7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java @@ -20,6 +20,8 @@ import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils; import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.SchemaItem; +import java.util.LinkedHashSet; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -137,7 +139,7 @@ public class DomainEntityService { chatFilter.setValue(String.valueOf(entity)); chatFilter.setOperator(FilterOperatorEnum.EQUALS); chatFilter.setBizName(getEntityPrimaryName(domainInfo)); - List chatFilters = new ArrayList<>(); + Set chatFilters = new LinkedHashSet(); chatFilters.add(chatFilter); semanticParseInfo.setDimensionFilters(chatFilters); @@ -167,8 +169,8 @@ public class DomainEntityService { } } - private List getDimensions(EntityInfo domainInfo) { - List dimensions = new ArrayList<>(); + private Set getDimensions(EntityInfo domainInfo) { + Set dimensions = new LinkedHashSet(); for (DataInfo mainEntityDimension : domainInfo.getDimensions()) { SchemaItem dimension = new SchemaItem(); dimension.setBizName(mainEntityDimension.getBizName()); @@ -186,8 +188,8 @@ public class DomainEntityService { return entryKey; } - private List getMetrics(EntityInfo domainInfo) { - List metrics = new ArrayList<>(); + private Set getMetrics(EntityInfo domainInfo) { + Set metrics = new LinkedHashSet(); for (DataInfo metricValue : domainInfo.getMetrics()) { SchemaItem metric = new SchemaItem(); metric.setBizName(metricValue.getBizName()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java index c999c1606..e9aed284f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java @@ -3,6 +3,8 @@ package com.tencent.supersonic.chat.application; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.response.QueryResultResp; @@ -18,6 +20,9 @@ import com.tencent.supersonic.chat.domain.service.QueryService; import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter; import com.tencent.supersonic.common.util.json.JsonUtil; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java index aeb791789..5ffd118de 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java @@ -4,8 +4,8 @@ import com.google.common.collect.Lists; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.request.QueryContextReq; -import com.tencent.supersonic.chat.api.service.SemanticLayer; import com.tencent.supersonic.chat.application.knowledge.NatureHelper; +import com.tencent.supersonic.chat.application.knowledge.WordNatureService; import com.tencent.supersonic.chat.application.mapper.SearchMatchStrategy; import com.tencent.supersonic.chat.domain.pojo.search.DomainInfoStat; import com.tencent.supersonic.chat.domain.pojo.search.DomainWithSemanticType; @@ -15,12 +15,10 @@ import com.tencent.supersonic.chat.domain.pojo.semantic.DomainInfos; import com.tencent.supersonic.chat.domain.service.ChatService; import com.tencent.supersonic.chat.domain.service.SearchService; import com.tencent.supersonic.chat.domain.utils.NatureConverter; -import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter; import com.tencent.supersonic.common.nlp.ItemDO; import com.tencent.supersonic.common.nlp.MapResult; import com.tencent.supersonic.common.nlp.NatureType; import com.tencent.supersonic.common.nlp.WordNature; -import com.tencent.supersonic.knowledge.application.online.BaseWordNature; import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper; import java.util.ArrayList; import java.util.Comparator; @@ -47,12 +45,11 @@ import org.springframework.stereotype.Service; @Service public class SearchServiceImpl implements SearchService { - private final Logger logger = LoggerFactory.getLogger(SearchServiceImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(SearchServiceImpl.class); @Autowired - private SemanticLayer semanticLayer; + private WordNatureService wordNatureService; @Autowired private ChatService chatService; - @Autowired private SearchMatchStrategy searchMatchStrategy; @@ -63,13 +60,16 @@ public class SearchServiceImpl implements SearchService { public List search(QueryContextReq queryCtx) { String queryText = queryCtx.getQueryText(); // 1.get meta info - DomainInfos domainInfosDb = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>())); + DomainInfos domainInfosDb = wordNatureService.getCache().getUnchecked(""); + List metricsDb = domainInfosDb.getMetrics(); final Map domainToName = domainInfosDb.getDomainToName(); // 2.detect by segment - List originals = HanlpHelper.getSegment().seg(queryText).stream().collect(Collectors.toList()); - Map> regTextMap = searchMatchStrategy.matchWithMatchText(queryText, originals); - + List originals = HanlpHelper.getSegment().seg(queryText.toLowerCase()).stream() + .collect(Collectors.toList()); + Map> regTextMap = searchMatchStrategy.matchWithMatchText(queryText, originals, + queryCtx.getDomainId()); + regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue())); // 3.get the most matching data Optional>> mostSimilarSearchResult = regTextMap.entrySet() .stream() @@ -77,28 +77,28 @@ public class SearchServiceImpl implements SearchService { .reduce((entry1, entry2) -> entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length() ? entry1 : entry2); - logger.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult); + LOGGER.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult); // 4.optimize the results after the query if (!mostSimilarSearchResult.isPresent()) { - logger.info("unable to find any information through search , queryCtx:{}", queryCtx); + LOGGER.info("unable to find any information through search , queryCtx:{}", queryCtx); return Lists.newArrayList(); } Map.Entry> searchTextEntry = mostSimilarSearchResult.get(); - logger.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx); + LOGGER.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx); Set searchResults = new LinkedHashSet(); DomainInfoStat domainStat = NatureHelper.getDomainStat(originals); - List possibleDomains = getPossibleDomains(queryCtx, originals, domainStat); + List possibleDomains = getPossibleDomains(queryCtx, originals, domainStat, queryCtx.getDomainId()); // 4.1 priority dimension metric boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDomains), domainToName, - searchTextEntry, - searchResults); + searchTextEntry, searchResults); // 4.2 process based on dimension values MatchText matchText = searchTextEntry.getKey(); - Map natureToNameMap = getNatureToNameMap(searchTextEntry); + Map natureToNameMap = getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDomains)); + LOGGER.debug("possibleDomains:{},natureToNameMap:{}", possibleDomains, natureToNameMap); for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) { searchDimensionValue(metricsDb, domainToName, domainStat.getMetricDomainCount(), searchResults, @@ -108,12 +108,19 @@ public class SearchServiceImpl implements SearchService { } private List getPossibleDomains(QueryContextReq queryCtx, List originals, - DomainInfoStat domainStat) { + DomainInfoStat domainStat, Integer webDomainId) { + + if (Objects.nonNull(webDomainId) && webDomainId > 0) { + List result = new ArrayList<>(); + result.add(webDomainId); + return result; + } + List possibleDomains = NatureHelper.selectPossibleDomains(originals); Long contextDomain = chatService.getContextDomain(queryCtx.getChatId()); - logger.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain); + LOGGER.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain); // If nothing is recognized or only metric are present, then add the contextDomain. if (nothingOrOnlyMetric(domainStat) && effectiveDomain(contextDomain)) { @@ -195,16 +202,25 @@ public class SearchServiceImpl implements SearchService { * @param recommendTextListEntry * @return */ - private Map getNatureToNameMap(Map.Entry> recommendTextListEntry) { + private Map getNatureToNameMap(Map.Entry> recommendTextListEntry, + Set possibleDomains) { List recommendValues = recommendTextListEntry.getValue(); return recommendValues.stream() - .flatMap(entry -> entry.getNatures().stream().map(nature -> { - WordNature posDO = new WordNature(); - posDO.setWord(entry.getName()); - posDO.setNature(nature); - return posDO; - } - )).sorted(Comparator.comparingInt(a -> a.getWord().length())) + .flatMap(entry -> entry.getNatures().stream() + .filter(nature -> { + if (CollectionUtils.isEmpty(possibleDomains)) { + return true; + } + Integer domain = NatureHelper.getDomain(nature); + return possibleDomains.contains(domain); + }) + .map(nature -> { + WordNature posDO = new WordNature(); + posDO.setWord(entry.getName()); + posDO.setNature(nature); + return posDO; + } + )).sorted(Comparator.comparingInt(a -> a.getWord().length())) .collect(Collectors.toMap(WordNature::getNature, WordNature::getWord, (value1, value2) -> value1, LinkedHashMap::new)); } @@ -233,7 +249,7 @@ public class SearchServiceImpl implements SearchService { domainToName.get(domain), domain, semanticType)); } } - logger.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult, + LOGGER.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult, dimensionMetricClassIds, possibleDomains); } return existMetric; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java index 938fd1bda..fa1630538 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java @@ -10,7 +10,6 @@ import org.springframework.context.ApplicationListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.util.ArrayList; import java.util.List; @Slf4j @@ -23,7 +22,6 @@ public class ApplicationStartedInit implements ApplicationListener preWordNatures = new ArrayList<>(); @Override public void onApplicationEvent(ApplicationStartedEvent event) { @@ -32,7 +30,7 @@ public class ApplicationStartedInit implements ApplicationListener wordNatures = wordNatureService.getAllWordNature(); - this.preWordNatures = wordNatures; + wordNatureService.setPreWordNatures(wordNatures); onlineKnowledgeService.reloadAllData(wordNatures); @@ -51,14 +49,15 @@ public class ApplicationStartedInit implements ApplicationListener wordNatures = wordNatureService.getAllWordNature(); + List preWordNatures = wordNatureService.getPreWordNatures(); if (CollectionUtils.isEqualCollection(wordNatures, preWordNatures)) { log.debug("wordNatures is not change, reloadKnowledge end"); return; } - this.preWordNatures = wordNatures; - + wordNatureService.setPreWordNatures(wordNatures); onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature()); + wordNatureService.getCache().refresh(""); } catch (Exception e) { log.error("reloadKnowledge error", e); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java index 8ac86e014..d8968f53b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; public class NatureHelper { private static final Logger LOGGER = LoggerFactory.getLogger(NatureHelper.class); + private static boolean isDomainOrEntity(Term term, Integer domain) { return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString() .endsWith(NatureType.ENTITY.getType()); @@ -96,7 +97,7 @@ public class NatureHelper { /** * Get the number of types of class parts of speech - * classId -> (nature , natureCount) + * domainId -> (nature , natureCount) * * @param terms * @return diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java index 5d073c337..d6236ba29 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.chat.application.knowledge; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.tencent.supersonic.chat.api.service.SemanticLayer; import com.tencent.supersonic.chat.domain.pojo.semantic.DomainInfos; import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter; @@ -9,6 +12,7 @@ import com.tencent.supersonic.common.nlp.WordNature; import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -21,10 +25,25 @@ import org.springframework.stereotype.Service; @Service public class WordNatureService { - private final Logger logger = LoggerFactory.getLogger(WordNatureService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(WordNatureService.class); @Autowired private SemanticLayer semanticLayer; + private static final Integer META_CACHE_TIME = 5; + + private List preWordNatures = new ArrayList<>(); + + private LoadingCache cache = CacheBuilder.newBuilder() + .expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES) + .build( + new CacheLoader() { + @Override + public DomainInfos load(String key) { + LOGGER.info("load getDomainSchemaInfo cache [{}]", key); + return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>())); + } + } + ); public List getAllWordNature() { @@ -45,7 +64,19 @@ public class WordNatureService { private void addNatureToResult(NatureType value, List metas, List natures) { List natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas); - logger.debug("nature type:{} , nature size:{}", value.name(), natureList.size()); + LOGGER.debug("nature type:{} , nature size:{}", value.name(), natureList.size()); natures.addAll(natureList); } + + public List getPreWordNatures() { + return preWordNatures; + } + + public void setPreWordNatures(List preWordNatures) { + this.preWordNatures = preWordNatures; + } + + public LoadingCache getCache() { + return cache; + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java index abd95004f..392e604af 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java @@ -12,6 +12,7 @@ import com.tencent.supersonic.common.nlp.MapResult; import com.tencent.supersonic.common.nlp.NatureType; import com.tencent.supersonic.common.util.context.ContextUtils; import com.tencent.supersonic.common.util.json.JsonUtil; +import com.tencent.supersonic.knowledge.application.online.BaseWordNature; import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory; import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper; import java.util.ArrayList; @@ -28,18 +29,22 @@ public class HanlpSchemaMapper implements SchemaMapper { private static final Logger LOGGER = LoggerFactory.getLogger(HanlpSchemaMapper.class); @Override - public void map(QueryContextReq searchCtx) { + public void map(QueryContextReq queryContext) { + + List terms = HanlpHelper.getSegment().seg(queryContext.getQueryText().toLowerCase()).stream() + .collect(Collectors.toList()); - List terms = HanlpHelper.getSegment().seg(searchCtx.getQueryText()).stream().collect(Collectors.toList()); terms.forEach( - item -> LOGGER.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(), item.frequency) + item -> LOGGER.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(), + item.getFrequency()) ); QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class); - List matches = matchStrategy.match(searchCtx.getQueryText(), terms, 0); - LOGGER.info("searchCtx:{},matches:{}", searchCtx, matches); + List matches = matchStrategy.match(queryContext.getQueryText(), terms, queryContext.getDomainId()); + HanlpHelper.transLetterOriginal(matches); + LOGGER.info("queryContext:{},matches:{}", queryContext, matches); - convertTermsToSchemaMapInfo(matches, searchCtx.getMapInfo()); + convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo()); } private void convertTermsToSchemaMapInfo(List mapResults, SchemaMapInfo schemaMap) { @@ -59,11 +64,14 @@ public class HanlpSchemaMapper implements SchemaMapper { SchemaElementMatch schemaElementMatch = new SchemaElementMatch(); schemaElementMatch.setElementType(elementType); - Integer elementID = WordNatureStrategyFactory.get(NatureType.getNatureType(nature)) - .getElementID(nature); + BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature)); + Integer elementID = baseWordNature.getElementID(nature); schemaElementMatch.setElementID(elementID); + Long frequency = baseWordNature.getFrequency(nature); + schemaElementMatch.setFrequency(frequency); schemaElementMatch.setWord(mapResult.getName()); schemaElementMatch.setSimilarity(mapResult.getSimilarity()); + schemaElementMatch.setDetectWord(mapResult.getDetectWord()); Map> domainElementMatches = schemaMap.getDomainElementMatches(); List schemaElementMatches = domainElementMatches.putIfAbsent(domain, diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/MatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/MatchStrategy.java index 7cd3cebd9..9e1348ae4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/MatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/MatchStrategy.java @@ -13,18 +13,10 @@ import java.util.Map; */ public interface MatchStrategy { - /*** - * match - * @param terms - * @return - */ - List match(String text, List terms, int retryCount); + List match(String text, List terms, Integer detectDomainId); - List match(String text, List terms, int retryCount, Integer detectDomainId); - - - Map> matchWithMatchText(String text, List originals); + Map> matchWithMatchText(String text, List originals, Integer detectDomainId); /*** * exist dimension values diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryMatchStrategy.java index 3196cb81e..11a689843 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryMatchStrategy.java @@ -40,35 +40,31 @@ public class QueryMatchStrategy implements MatchStrategy { private Double dimensionValueThresholdConfig; @Override - public List match(String text, List terms, int retryCount) { - return match(text, terms, retryCount, null); - } - - @Override - public List match(String text, List terms, int retryCount, Integer detectDomainId) { + public List match(String text, List terms, Integer detectDomainId) { if (CollectionUtils.isEmpty(terms) || StringUtils.isEmpty(text)) { return null; } Map regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length)) .collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2)); + List offsetList = terms.stream().sorted(Comparator.comparing(Term::getOffset)) .map(term -> term.getOffset()).collect(Collectors.toList()); - LOGGER.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", retryCount, terms, - regOffsetToLength, offsetList, - detectDomainId); + LOGGER.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", terms, + regOffsetToLength, offsetList, detectDomainId); - return detect(text, regOffsetToLength, offsetList, detectDomainId, retryCount); + return detect(text, regOffsetToLength, offsetList, detectDomainId); } @Override - public Map> matchWithMatchText(String text, List originals) { + public Map> matchWithMatchText(String text, List originals, + Integer detectDomainId) { return null; } private List detect(String text, Map regOffsetToLength, List offsetList, - Integer detectDomainId, int retryCount) { + Integer detectDomainId) { List results = Lists.newArrayList(); for (Integer index = 0; index <= text.length() - 1; ) { @@ -79,7 +75,7 @@ public class QueryMatchStrategy implements MatchStrategy { int offset = getStepOffset(offsetList, index); i = getStepIndex(regOffsetToLength, i); if (i <= text.length()) { - List mapResults = detectByStep(text, detectDomainId, index, i, offset, retryCount); + List mapResults = detectByStep(text, detectDomainId, index, i, offset); mapResultRowSet.addAll(mapResults); } } @@ -90,8 +86,7 @@ public class QueryMatchStrategy implements MatchStrategy { return results; } - private List detectByStep(String text, Integer detectClassId, Integer index, Integer i, int offset, - int retryCount) { + private List detectByStep(String text, Integer detectDomainId, Integer index, Integer i, int offset) { String detectSegment = text.substring(index, i); // step1. pre search LinkedHashSet mapResults = Suggester.prefixSearch(detectSegment, oneDetectionMaxSize) @@ -109,11 +104,11 @@ public class QueryMatchStrategy implements MatchStrategy { mapResults = mapResults.stream().sorted((a, b) -> -(b.getName().length() - a.getName().length())) .collect(Collectors.toCollection(LinkedHashSet::new)); // step4. filter by classId - if (Objects.nonNull(detectClassId) && detectClassId > 0) { + if (Objects.nonNull(detectDomainId) && detectDomainId > 0) { LOGGER.debug("detectDomainId:{}, before parseResults:{}", mapResults); mapResults = mapResults.stream().map(entry -> { List natures = entry.getNatures().stream().filter( - nature -> nature.startsWith(NatureType.NATURE_SPILT + detectClassId) || (nature.startsWith( + nature -> nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) || (nature.startsWith( NatureType.NATURE_SPILT)) ).collect(Collectors.toList()); entry.setNatures(natures); @@ -123,8 +118,7 @@ public class QueryMatchStrategy implements MatchStrategy { } // step5. filter by similarity mapResults = mapResults.stream() - .filter(term -> getSimilarity(detectSegment, term.getName()) >= getThresholdMatch(term.getNatures(), - retryCount)) + .filter(term -> getSimilarity(detectSegment, term.getName()) >= getThresholdMatch(term.getNatures())) .filter(term -> CollectionUtils.isNotEmpty(term.getNatures())) .collect(Collectors.toCollection(LinkedHashSet::new)); @@ -170,11 +164,11 @@ public class QueryMatchStrategy implements MatchStrategy { return index; } - private double getThresholdMatch(List natures, int retryCount) { + private double getThresholdMatch(List natures) { if (existDimensionValues(natures)) { return dimensionValueThresholdConfig; } - return metricDimensionThresholdConfig - STEP * retryCount; + return metricDimensionThresholdConfig; } } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/SearchMatchStrategy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/SearchMatchStrategy.java index f7d2c7da8..8e7c57013 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/SearchMatchStrategy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/SearchMatchStrategy.java @@ -23,20 +23,16 @@ public class SearchMatchStrategy implements MatchStrategy { private static final int SEARCH_SIZE = 3; + @Override - public List match(String text, List terms, int retryCount) { + public List match(String text, List terms, Integer detectDomainId) { return null; } @Override - public List match(String text, List terms, int retryCount, Integer detectDomainId) { - - return null; - } - - @Override - public Map> matchWithMatchText(String text, List originals) { + public Map> matchWithMatchText(String text, List originals, + Integer detectDomainId) { Map regOffsetToLength = originals.stream() .filter(entry -> !entry.nature.toString().startsWith(NatureType.NATURE_SPILT)) @@ -70,7 +66,17 @@ public class SearchMatchStrategy implements MatchStrategy { mapResults = mapResults.stream().filter(entry -> { List natures = entry.getNatures().stream() .filter(nature -> !nature.endsWith(NatureType.ENTITY.getType())) - .collect(Collectors.toList()); + .filter(nature -> { + if (Objects.isNull(detectDomainId) || detectDomainId <= 0) { + return true; + } + if (nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) + && nature.startsWith(NatureType.NATURE_SPILT)) { + return true; + } + return false; + } + ).collect(Collectors.toList()); if (CollectionUtils.isEmpty(natures)) { return false; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java index cb79eb5b5..6a6c9d092 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.common.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.SchemaItem; import com.tencent.supersonic.common.util.context.ContextUtils; import java.util.List; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -21,7 +22,7 @@ import org.springframework.stereotype.Component; public class AggregateSemanticParser implements SemanticParser { private final Logger logger = LoggerFactory.getLogger(AggregateSemanticParser.class); - public static final Integer TOPN_LIMIT = 10; + public static final Integer TOPN_LIMIT = 1000; private AggregateTypeResolver aggregateTypeResolver; @@ -33,7 +34,7 @@ public class AggregateSemanticParser implements SemanticParser { SemanticParseInfo semanticParse = queryContext.getParseInfo(); - List metrics = semanticParse.getMetrics(); + Set metrics = semanticParse.getMetrics(); semanticParse.setNativeQuery(getNativeQuery(aggregateType, queryContext)); @@ -47,12 +48,12 @@ public class AggregateSemanticParser implements SemanticParser { /** * query mode reset by the AggregateType * - * @param searchCtx + * @param queryContext * @param aggregateType */ - private void resetQueryModeByAggregateType(QueryContextReq searchCtx, AggregateTypeEnum aggregateType) { + private void resetQueryModeByAggregateType(QueryContextReq queryContext, AggregateTypeEnum aggregateType) { - SemanticParseInfo parseInfo = searchCtx.getParseInfo(); + SemanticParseInfo parseInfo = queryContext.getParseInfo(); String queryMode = parseInfo.getQueryMode(); if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) { if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType) @@ -63,7 +64,7 @@ public class AggregateSemanticParser implements SemanticParser { } } if (MetricFilter.QUERY_MODE.equals(queryMode) || MetricCompare.QUERY_MODE.equals(queryMode)) { - if (aggregateTypeResolver.hasCompareIntentionalWords(searchCtx.getQueryText())) { + if (aggregateTypeResolver.hasCompareIntentionalWords(queryContext.getQueryText())) { parseInfo.setQueryMode(MetricCompare.QUERY_MODE); } else { parseInfo.setQueryMode(MetricFilter.QUERY_MODE); @@ -72,11 +73,11 @@ public class AggregateSemanticParser implements SemanticParser { logger.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode()); } - private boolean getNativeQuery(AggregateTypeEnum aggregateType, QueryContextReq searchCtx) { + private boolean getNativeQuery(AggregateTypeEnum aggregateType, QueryContextReq queryContext) { if (AggregateTypeEnum.TOPN.equals(aggregateType)) { return true; } - return searchCtx.getParseInfo().getNativeQuery(); + return queryContext.getParseInfo().getNativeQuery(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DefaultMetricSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DefaultMetricSemanticParser.java index 5d847e19f..2d8036ecf 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DefaultMetricSemanticParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DefaultMetricSemanticParser.java @@ -9,8 +9,6 @@ import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.service.SemanticParser; -import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp; -import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp; import com.tencent.supersonic.chat.application.parser.resolver.DomainResolver; import com.tencent.supersonic.chat.application.query.EntityDetail; import com.tencent.supersonic.chat.application.query.EntityListFilter; @@ -24,8 +22,10 @@ import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.SchemaItem; import com.tencent.supersonic.common.util.context.ContextUtils; -import java.util.ArrayList; +import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp; +import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -104,33 +104,39 @@ public class DefaultMetricSemanticParser implements SemanticParser { return primaryDimensions; } - protected void addEntityDetailDimensionMetric(QueryContextReq searchCtx, ChatContext chatCtx) { - if (searchCtx.getParseInfo().getDomainId() > 0) { - ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo( - searchCtx.getParseInfo().getDomainId()); + protected void addEntityDetailDimensionMetric(QueryContextReq queryContext, ChatContext chatCtx) { + if (queryContext.getParseInfo().getDomainId() > 0) { + Long domainId = queryContext.getParseInfo().getDomainId(); + ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domainId); if (chaConfigRichDesc != null) { - SemanticParseInfo semanticParseInfo = searchCtx.getParseInfo(); - if (Objects.nonNull(semanticParseInfo) && CollectionUtils.isEmpty(semanticParseInfo.getDimensions())) { - List dimensions = new ArrayList<>(); - List metrics = new ArrayList<>(); - if (chaConfigRichDesc.getEntity() != null - && chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) { - chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream() - .forEach(m -> metrics.add(getMetric(m))); - chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream() - .forEach(m -> dimensions.add(getDimension(m))); - } - semanticParseInfo.setDimensions(dimensions); - semanticParseInfo.setMetrics(metrics); + if (chaConfigRichDesc.getEntity() == null + || chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() == null) { + return; } + SemanticParseInfo semanticParseInfo = queryContext.getParseInfo(); + Set metrics = new LinkedHashSet(); + chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream() + .forEach(m -> metrics.add(getMetric(m))); + semanticParseInfo.setMetrics(metrics); + List schemaElementMatches = queryContext.getMapInfo() + .getMatchedElements(domainId.intValue()); + if (CollectionUtils.isEmpty(schemaElementMatches) || schemaElementMatches.stream() + .filter(s -> SchemaElementType.DIMENSION.equals(s.getElementType())).count() <= 0) { + logger.info("addEntityDetailDimensionMetric catch"); + Set dimensions = new LinkedHashSet(); + chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream() + .forEach(m -> dimensions.add(getDimension(m))); + semanticParseInfo.setDimensions(dimensions); + + } } } } - protected void defaultQueryMode(QueryContextReq searchCtx, ChatContext chatCtx) { - SchemaMapInfo schemaMap = searchCtx.getMapInfo(); - SemanticParseInfo parseInfo = searchCtx.getParseInfo(); + protected void defaultQueryMode(QueryContextReq queryContext, ChatContext chatCtx) { + SchemaMapInfo schemaMap = queryContext.getMapInfo(); + SemanticParseInfo parseInfo = queryContext.getParseInfo(); if (StringUtils.isEmpty(parseInfo.getQueryMode())) { if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) { // @@ -182,12 +188,12 @@ public class DefaultMetricSemanticParser implements SemanticParser { } - private void fillDateDomain(ChatContext chatCtx, QueryContextReq searchCtx) { - SemanticParseInfo parseInfo = searchCtx.getParseInfo(); + private void fillDateDomain(ChatContext chatCtx, QueryContextReq queryContext) { + SemanticParseInfo parseInfo = queryContext.getParseInfo(); if (parseInfo == null || parseInfo.getDateInfo() == null) { boolean isUpdateTime = false; - if (selectStrategy.isDomainSwitch(chatCtx, searchCtx)) { + if (selectStrategy.isDomainSwitch(chatCtx, queryContext)) { isUpdateTime = true; } if (chatCtx.getParseInfo() == null @@ -212,7 +218,7 @@ public class DefaultMetricSemanticParser implements SemanticParser { if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) && CollectionUtils.isEmpty( semanticParseInfo.getDimensions())) { - List metrics = new ArrayList<>(); + Set metrics = new LinkedHashSet(); chaConfigRichDesc.getDefaultMetrics().stream().forEach(metric -> { SchemaItem metricTmp = new SchemaItem(); metricTmp.setId(metric.getMetricId()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java index 26c23a383..565c397ef 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java @@ -25,6 +25,7 @@ import java.util.Objects; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.io.support.SpringFactoriesLoader; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -32,10 +33,15 @@ import org.springframework.util.CollectionUtils; public class DomainSemanticParser implements SemanticParser { private final Logger logger = LoggerFactory.getLogger(DomainSemanticParser.class); - private DomainResolver domainResolver; + private List domainResolverList; private SemanticQueryResolver semanticQueryResolver; + public DomainSemanticParser() { + domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class, + Thread.currentThread().getContextClassLoader()); + } + @Override public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) { DomainInfos domainInfosDb = SchemaInfoConverter.convert( @@ -45,7 +51,7 @@ public class DomainSemanticParser implements SemanticParser { SchemaMapInfo mapInfo = queryContext.getMapInfo(); SemanticParseInfo parseInfo = queryContext.getParseInfo(); - domainResolver = ContextUtils.getBean(DomainResolver.class); + //domainResolver = ContextUtils.getBean(DomainResolver.class); semanticQueryResolver = ContextUtils.getBean(SemanticQueryResolver.class); Map domainSemanticQuery = new HashMap<>(); @@ -72,15 +78,19 @@ public class DomainSemanticParser implements SemanticParser { } } else if (domainSemanticQuery.size() > 1) { // will choose one by the domain select - Integer domainId = domainResolver.resolve(domainSemanticQuery, queryContext, chatCtx, mapInfo); - if (domainId > 0) { - Map.Entry match = domainSemanticQuery.entrySet().stream() - .filter(entry -> entry.getKey().equals(domainId)).findFirst().orElse(null); - logger.info("select by selectStrategy [{}:{}]", domainId, match.getValue()); - parseInfo.setDomainId(Long.valueOf(match.getKey())); - parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.getKey()))); - parseInfo.setQueryMode(match.getValue().getQueryMode()); - return false; + Optional domainId = domainResolverList.stream() + .map(domainResolver -> domainResolver.resolve(domainSemanticQuery, queryContext, chatCtx, mapInfo)) + .filter(d -> d > 0).findFirst(); + if (domainId.isPresent() && domainId.get() > 0) { + for (Map.Entry match : domainSemanticQuery.entrySet()) { + if (match.getKey().equals(domainId.get())) { + logger.info("select by selectStrategy [{}:{}]", domainId.get(), match.getValue()); + parseInfo.setDomainId(Long.valueOf(match.getKey())); + parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.getKey()))); + parseInfo.setQueryMode(match.getValue().getQueryMode()); + return false; + } + } } } // Round 2: no domains can be found yet, count in chat context diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/ListFilterParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/ListFilterParser.java index 192706fb2..a3f7a1e4f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/ListFilterParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/ListFilterParser.java @@ -19,6 +19,7 @@ import com.tencent.supersonic.common.pojo.SchemaItem; import com.tencent.supersonic.common.util.context.ContextUtils; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -43,7 +44,7 @@ public class ListFilterParser implements SemanticParser { this.fillDateEntityFilter(queryContext.getParseInfo()); this.addEntityDetailAndOrderByMetric(queryContext, chatCtx); this.dealNativeQuery(queryContext, true); - return false; + return true; } @@ -56,15 +57,15 @@ public class ListFilterParser implements SemanticParser { semanticParseInfo.setDateInfo(dateInfo); } - private void addEntityDetailAndOrderByMetric(QueryContextReq searchCtx, ChatContext chatCtx) { - if (searchCtx.getParseInfo().getDomainId() > 0L) { + private void addEntityDetailAndOrderByMetric(QueryContextReq queryContext, ChatContext chatCtx) { + if (queryContext.getParseInfo().getDomainId() > 0L) { ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo( - searchCtx.getParseInfo().getDomainId()); + queryContext.getParseInfo().getDomainId()); if (chaConfigRichDesc != null) { - SemanticParseInfo semanticParseInfo = searchCtx.getParseInfo(); - List dimensions = new ArrayList(); + SemanticParseInfo semanticParseInfo = queryContext.getParseInfo(); + Set dimensions = new LinkedHashSet(); Set primaryDimensions = this.addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions); - List metrics = new ArrayList(); + Set metrics = new LinkedHashSet(); if (chaConfigRichDesc.getEntity() != null && chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) { chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream() @@ -76,7 +77,7 @@ public class ListFilterParser implements SemanticParser { semanticParseInfo.setDimensions(dimensions); semanticParseInfo.setMetrics(metrics); - List orders = new ArrayList(); + Set orders = new LinkedHashSet(); if (chaConfigRichDesc.getEntity() != null && chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) { chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream() @@ -89,7 +90,7 @@ public class ListFilterParser implements SemanticParser { } - private Set addPrimaryDimension(EntityRichInfo entity, List dimensions) { + private Set addPrimaryDimension(EntityRichInfo entity, Set dimensions) { Set primaryDimensions = new HashSet(); if (!Objects.isNull(entity) && !CollectionUtils.isEmpty(entity.getEntityIds())) { entity.getEntityIds().stream().forEach((dimSchemaDesc) -> { @@ -120,9 +121,9 @@ public class ListFilterParser implements SemanticParser { return queryMeta; } - private void dealNativeQuery(QueryContextReq searchCtx, boolean isNativeQuery) { - if (Objects.nonNull(searchCtx) && Objects.nonNull(searchCtx.getParseInfo())) { - searchCtx.getParseInfo().setNativeQuery(isNativeQuery); + private void dealNativeQuery(QueryContextReq queryContext, boolean isNativeQuery) { + if (Objects.nonNull(queryContext) && Objects.nonNull(queryContext.getParseInfo())) { + queryContext.getParseInfo().setNativeQuery(isNativeQuery); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/BaseDomainResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/BaseDomainResolver.java new file mode 100644 index 000000000..62edf7521 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/BaseDomainResolver.java @@ -0,0 +1,131 @@ +package com.tencent.supersonic.chat.application.parser.resolver; + +import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElementCount; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; +import com.tencent.supersonic.chat.api.request.QueryContextReq; +import com.tencent.supersonic.chat.api.service.SemanticQuery; +import com.tencent.supersonic.chat.domain.utils.ContextHelper; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class BaseDomainResolver implements DomainResolver { + + @Override + public boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq searchCtx) { + Long contextDomain = chatCtx.getParseInfo().getDomainId(); + Long currentDomain = searchCtx.getParseInfo().getDomainId(); + boolean noSwitch = currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain); + log.info("ChatContext isDomainSwitch [{}]", !noSwitch); + return !noSwitch; + } + + public abstract Integer selectDomain(Map domainQueryModes, QueryContextReq searchCtx, + ChatContext chatCtx, SchemaMapInfo schemaMap); + + @Override + public Integer resolve(Map domainQueryModes, QueryContextReq searchCtx, + ChatContext chatCtx, SchemaMapInfo schemaMap) { + Integer selectDomain = selectDomain(domainQueryModes, searchCtx, chatCtx, schemaMap); + if (selectDomain > 0) { + log.info("selectDomain {} ", selectDomain); + return selectDomain; + } + // get the max SchemaElementType number + return selectDomainBySchemaElementCount(domainQueryModes, schemaMap); + } + + protected static Integer selectDomainBySchemaElementCount(Map domainQueryModes, + SchemaMapInfo schemaMap) { + Map domainTypeMap = getDomainTypeMap(schemaMap); + if (domainTypeMap.size() == 1) { + Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey(); + if (domainQueryModes.containsKey(domainSelect)) { + log.info("selectDomain from domainTypeMap not order [{}]", domainSelect); + return domainSelect; + } + } else { + Map.Entry maxDomain = domainTypeMap.entrySet().stream() + .filter(entry -> domainQueryModes.containsKey(entry.getKey())) + .sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null); + if (maxDomain != null) { + log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey()); + return maxDomain.getKey(); + } + } + return 0; + } + + + /** + * to check can switch domain if context exit domain + * + * @return false will use context domain, true will use other domain , maybe include context domain + */ + protected static boolean isAllowSwitch(Map domainQueryModes, SchemaMapInfo schemaMap, + ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) { + if (!Objects.nonNull(domainId) || domainId <= 0) { + return true; + } + // except content domain, calculate the number of types for each domain, if numbers<=1 will not switch + Map domainTypeMap = getDomainTypeMap(schemaMap); + log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap); + long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream() + .filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId)) + .filter(entry -> entry.getValue().getCount() > 1).count(); + if (otherDomainTypeNumBigOneCount >= 1) { + return true; + } + // if query text only contain time , will not switch + if (searchCtx.getQueryText() != null && searchCtx.getParseInfo().getDateInfo() != null) { + if (searchCtx.getParseInfo().getDateInfo().getText() != null) { + if (searchCtx.getParseInfo().getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) { + log.info("timeParseResults is not null , can not switch context , timeParseResults:{},", + searchCtx.getParseInfo().getDateInfo()); + return false; + } + } + } + // if context domain not in schemaMap , will switch + if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) { + log.info("domainId not in schemaMap "); + return true; + } + // other will not switch + return false; + } + + protected static Map getDomainTypeMap(SchemaMapInfo schemaMap) { + Map domainCount = new HashMap<>(); + for (Map.Entry> entry : schemaMap.getDomainElementMatches().entrySet()) { + List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); + if (schemaElementMatches != null && schemaElementMatches.size() > 0) { + if (!domainCount.containsKey(entry.getKey())) { + domainCount.put(entry.getKey(), new SchemaElementCount()); + } + SchemaElementCount schemaElementCount = domainCount.get(entry.getKey()); + Set schemaElementTypes = new HashSet<>(); + schemaElementMatches.stream() + .forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType())); + SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() + .sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null); + if (schemaElementMatchMax != null) { + schemaElementCount.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); + } + schemaElementCount.setCount(schemaElementTypes.size()); + + } + } + return domainCount; + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/HeuristicDomainResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/HeuristicDomainResolver.java index cff3c10fa..12ec57684 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/HeuristicDomainResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/HeuristicDomainResolver.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.service.SemanticQuery; import com.tencent.supersonic.chat.domain.utils.ContextHelper; +import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Primary; @@ -22,121 +23,28 @@ import java.util.Set; import java.util.stream.Collectors; -@Service("heuristicDomainResolver") -@Primary -public class HeuristicDomainResolver implements DomainResolver { - - private static final Logger LOGGER = LoggerFactory.getLogger(HeuristicDomainResolver.class); +@Service("DomainResolver") +@Slf4j +public class HeuristicDomainResolver extends BaseDomainResolver { @Override - public Integer resolve(Map domainQueryModes, QueryContextReq searchCtx, - ChatContext chatCtx, SchemaMapInfo schemaMap) { + public Integer selectDomain(Map domainQueryModes, QueryContextReq searchCtx, + ChatContext chatCtx, + SchemaMapInfo schemaMap) { // if QueryContext has domainId and in domainQueryModes if (domainQueryModes.containsKey(searchCtx.getDomainId())) { - LOGGER.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId()); + log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId()); return searchCtx.getDomainId(); } // if ChatContext has domainId and in domainQueryModes if (chatCtx.getParseInfo().getDomainId() > 0) { Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue()); if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) { - LOGGER.info("selectDomain from ChatContext [{}]", domainId); + log.info("selectDomain from ChatContext [{}]", domainId); return domainId; } } - // get the max SchemaElementType number - Map domainTypeMap = getDomainTypeMap(schemaMap); - if (domainTypeMap.size() == 1) { - Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey(); - if (domainQueryModes.containsKey(domainSelect)) { - LOGGER.info("selectDomain from domainTypeMap not order [{}]", domainSelect); - return domainSelect; - } - } else { - Map.Entry maxDomain = domainTypeMap.entrySet().stream() - .filter(entry -> domainQueryModes.containsKey(entry.getKey())) - .sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null); - if (maxDomain != null) { - LOGGER.info("selectDomain from domainTypeMap need order [{}]", maxDomain.getKey()); - return maxDomain.getKey(); - } - } - // bad case , here will not reach , default 0 - LOGGER.error("selectDomain not found "); + // default 0 return 0; } - - @Override - public boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq searchCtx) { - Long contextDomain = chatCtx.getParseInfo().getDomainId(); - Long currentDomain = searchCtx.getParseInfo().getDomainId(); - boolean noSwitch = currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain); - LOGGER.info("ChatContext isDomainSwitch [{}]", !noSwitch); - return !noSwitch; - } - - - /** - * to check can switch domain if context exit domain - * - * @return false will use context domain, true will use other domain , maybe include context domain - */ - private static boolean isAllowSwitch(Map domainQueryModes, SchemaMapInfo schemaMap, - ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) { - if (!Objects.nonNull(domainId) || domainId <= 0) { - return true; - } - // except content domain, calculate the number of types for each domain, if numbers<=1 will not switch - Map domainTypeMap = getDomainTypeMap(schemaMap); - LOGGER.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap); - long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream() - .filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId)) - .filter(entry -> entry.getValue().getCount() > 1).count(); - if (otherDomainTypeNumBigOneCount >= 1) { - return true; - } - // if query text only contain time , will not switch - if (searchCtx.getQueryText() != null && searchCtx.getParseInfo().getDateInfo() != null) { - if (searchCtx.getParseInfo().getDateInfo().getText() != null) { - if (searchCtx.getParseInfo().getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) { - LOGGER.info("timeParseResults is not null , can not switch context , timeParseResults:{},", - searchCtx.getParseInfo().getDateInfo()); - return false; - } - } - } - // if context domain not in schemaMap , will switch - if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) { - LOGGER.info("domainId not in schemaMap "); - return true; - } - // other will not switch - return false; - } - - private static Map getDomainTypeMap(SchemaMapInfo schemaMap) { - Map domainCount = new HashMap<>(); - for (Map.Entry> entry : schemaMap.getDomainElementMatches().entrySet()) { - List schemaElementMatches = schemaMap.getMatchedElements(entry.getKey()); - if (schemaElementMatches != null && schemaElementMatches.size() > 0) { - if (!domainCount.containsKey(entry.getKey())) { - domainCount.put(entry.getKey(), new SchemaElementCount()); - } - SchemaElementCount schemaElementCount = domainCount.get(entry.getKey()); - Set schemaElementTypes = new HashSet<>(); - schemaElementMatches.stream() - .forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType())); - SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream() - .sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null); - if (schemaElementMatchMax != null) { - schemaElementCount.setMaxSimilarity(schemaElementMatchMax.getSimilarity()); - } - schemaElementCount.setCount(schemaElementTypes.size()); - - } - } - return domainCount; - } - - } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/RegexAggregateTypeResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/RegexAggregateTypeResolver.java index 27fca2154..80a5d7647 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/RegexAggregateTypeResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/RegexAggregateTypeResolver.java @@ -22,7 +22,7 @@ public class RegexAggregateTypeResolver implements AggregateTypeResolver { aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高)")); aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低)")); aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)")); - aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|平均|avg)")); + aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)")); aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)")); aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)")); aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)")); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java index e86d6d42e..06e3d3999 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java @@ -11,6 +11,7 @@ import org.springframework.stereotype.Service; public class EntityListFilter extends BaseSemanticQuery { public static String QUERY_MODE = "ENTITY_LIST_FILTER"; + private static Long entityListLimit = 200L; public EntityListFilter() { queryModeOption.setAggregation(QueryModeElementOption.unused()); @@ -42,6 +43,7 @@ public class EntityListFilter extends BaseSemanticQuery { public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) { SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo(); ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters()); + semanticParseInfo.setLimit(entityListLimit); return semanticParseInfo; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java index 6d7a28c20..96d7189be 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java @@ -1,13 +1,24 @@ package com.tencent.supersonic.chat.application.query; import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.Filter; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption; import com.tencent.supersonic.chat.domain.utils.ContextHelper; +import com.tencent.supersonic.common.pojo.SchemaItem; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @Service +@Slf4j public class MetricCompare extends BaseSemanticQuery { public static String QUERY_MODE = "METRIC_COMPARE"; @@ -48,7 +59,61 @@ public class MetricCompare extends BaseSemanticQuery { SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo(); ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo); ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics()); - ContextHelper.appendList(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters()); + mergeAppend(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters()); + addCompareDimension(semanticParseInfo); return semanticParseInfo; } + + private void addCompareDimension(SemanticParseInfo semanticParseInfo) { + if (!semanticParseInfo.getDimensionFilters().isEmpty()) { + Set dimensions = semanticParseInfo.getDimensions().stream().map(d -> d.getBizName()).collect( + Collectors.toSet()); + log.info("addCompareDimension before [{}]", dimensions); + semanticParseInfo.getDimensionFilters().stream().filter(d -> d.getOperator().equals(FilterOperatorEnum.IN)) + .forEach( + d -> { + if (!dimensions.contains(d.getBizName())) { + SchemaItem schemaItem = new SchemaItem(); + schemaItem.setBizName(d.getBizName()); + schemaItem.setId(d.getElementID()); + semanticParseInfo.getDimensions().add(schemaItem); + dimensions.add(d.getBizName()); + } + } + ); + log.info("addCompareDimension after [{}]", dimensions); + } + } + + private void mergeAppend(Set from, Set to) { + if (!from.isEmpty()) { + for (Filter filter : from) { + if (FilterOperatorEnum.EQUALS.equals(filter.getOperator()) || FilterOperatorEnum.IN.equals( + filter.getOperator())) { + Optional toAdd = to.stream() + .filter(t -> t.getBizName().equalsIgnoreCase(filter.getBizName())).findFirst(); + if (toAdd.isPresent()) { + if (FilterOperatorEnum.EQUALS.equals(toAdd.get().getOperator()) || FilterOperatorEnum.IN.equals( + toAdd.get().getOperator())) { + List vals = new ArrayList<>(); + if (toAdd.get().getOperator().equals(FilterOperatorEnum.IN)) { + vals.addAll((List) (toAdd.get().getValue())); + } else { + vals.add(toAdd.get().getValue()); + } + if (filter.getOperator().equals(FilterOperatorEnum.IN)) { + vals.addAll((List) (filter.getValue())); + } else { + vals.add(filter.getValue()); + } + toAdd.get().setValue(vals); + toAdd.get().setOperator(FilterOperatorEnum.IN); + continue; + } + } + } + to.add(filter); + } + } + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java index b8667cc3f..792e1bde9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java @@ -2,21 +2,25 @@ package com.tencent.supersonic.chat.domain.pojo.chat; import com.tencent.supersonic.chat.api.pojo.Filter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.SchemaItem; -import java.util.ArrayList; -import java.util.List; import lombok.Data; @Data public class QueryData { Long domainId = 0L; - List metrics = new ArrayList<>(); - List dimensions = new ArrayList<>(); - List filters = new ArrayList<>(); - private List orders = new ArrayList<>(); + Set metrics = new HashSet<>(); + Set dimensions = new HashSet<>(); + Set dimensionFilters = new HashSet<>(); + Set metricFilters = new HashSet<>(); + private Set orders = new HashSet<>(); private DateConf dateInfo; private Long limit; private Boolean nativeQuery = false; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java index 2730cef29..cb5b9b4b3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java @@ -4,6 +4,7 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.response.QueryResultResp; +import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO; import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq; @@ -13,4 +14,7 @@ public interface ChatQueryRepository { void createChatQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx); + ChatQueryDO getLastChatQuery(long chatId); + + int updateChatQuery(ChatQueryDO chatQueryDO); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ChatService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ChatService.java index 273b82b4f..79dd8ceb3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ChatService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ChatService.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.request.QueryContextReq; import com.tencent.supersonic.chat.api.response.QueryResultResp; import com.tencent.supersonic.chat.domain.dataobject.ChatDO; +import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO; import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq; import java.util.List; @@ -40,4 +41,8 @@ public interface ChatService { PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId); public void addQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx); + + public ChatQueryDO getLastQuery(long chatId); + + public int updateQuery(ChatQueryDO chatQueryDO); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/ContextHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/ContextHelper.java index 498878503..9ceeb84fd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/ContextHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/ContextHelper.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.api.service.SemanticQuery; import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp; import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo; import com.tencent.supersonic.common.pojo.SchemaItem; +import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -71,7 +72,7 @@ public class ContextHelper { * @param from * @param to */ - public static void addIfEmpty(List from, List to) { + public static void addIfEmpty(Set from, Set to) { if (to.isEmpty() && !from.isEmpty()) { to.addAll(from); } @@ -82,7 +83,7 @@ public class ContextHelper { * @param from * @param to */ - public static void appendList(List from, List to) { + public static void appendList(Set from, Set to) { if (!from.isEmpty()) { to.addAll(from); } @@ -94,7 +95,7 @@ public class ContextHelper { * @param from * @param to */ - public static void updateList(List from, List to) { + public static void updateList(Set from, Set to) { if (!from.isEmpty()) { to.clear(); to.addAll(from); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/SchemaInfoConverter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/SchemaInfoConverter.java index 6b7a3403c..cec704546 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/SchemaInfoConverter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/SchemaInfoConverter.java @@ -21,9 +21,14 @@ import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp; import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; + import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; + +import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; public class SchemaInfoConverter { @@ -40,6 +45,7 @@ public class SchemaInfoConverter { queryStructCmd.setDateInfo(parseInfo.getDateInfo()); List dimensionFilters = parseInfo.getDimensionFilters().stream() + .filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName())) .map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue())) .collect(Collectors.toList()); queryStructCmd.setDimensionFilters(dimensionFilters); @@ -54,12 +60,13 @@ public class SchemaInfoConverter { .collect(Collectors.toList()); queryStructCmd.setGroups(dimensions); queryStructCmd.setLimit(parseInfo.getLimit()); - queryStructCmd.setOrders(getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics())); + Set order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics()); + queryStructCmd.setOrders(new ArrayList<>(order)); queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getMetrics(), parseInfo.getAggType())); return queryStructCmd; } - private static List getAggregatorByMetric(List metrics, AggregateTypeEnum aggregateType) { + private static List getAggregatorByMetric(Set metrics, AggregateTypeEnum aggregateType) { List aggregators = new ArrayList<>(); String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? "" : aggregateType.name(); @@ -157,11 +164,11 @@ public class SchemaInfoConverter { return result; } - public static List getOrder(List parseOrder, AggregateTypeEnum aggregator, List metrics) { + public static Set getOrder(Set parseOrder, AggregateTypeEnum aggregator, Set metrics) { if (!CollectionUtils.isEmpty(parseOrder)) { return parseOrder; } - List orders = new ArrayList<>(); + Set orders = new LinkedHashSet(); if (CollectionUtils.isEmpty(metrics)) { return orders; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/infrastructure/repository/ChatQueryRepositoryImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/infrastructure/repository/ChatQueryRepositoryImpl.java index 098fe1a68..f8743b543 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/infrastructure/repository/ChatQueryRepositoryImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/infrastructure/repository/ChatQueryRepositoryImpl.java @@ -14,11 +14,13 @@ import com.tencent.supersonic.chat.domain.repository.ChatQueryRepository; import com.tencent.supersonic.chat.infrastructure.mapper.ChatQueryDOMapper; import com.tencent.supersonic.common.util.json.JsonUtil; import com.tencent.supersonic.common.util.mybatis.PageUtils; +import java.util.List; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Repository; +import org.springframework.util.CollectionUtils; @Repository @Primary @@ -69,4 +71,24 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { Long queryId = Long.valueOf(chatQueryDOMapper.insert(chatQueryDO)); queryResponse.setQueryId(queryId); } + + @Override + public ChatQueryDO getLastChatQuery(long chatId) { + ChatQueryDOExample example = new ChatQueryDOExample(); + example.setOrderByClause("question_id desc"); + example.setLimitEnd(1); + example.setLimitStart(0); + Criteria criteria = example.createCriteria(); + criteria.andChatIdEqualTo(chatId); + List chatQueryDOS = chatQueryDOMapper.selectByExampleWithBLOBs(example); + if (!CollectionUtils.isEmpty(chatQueryDOS)) { + return chatQueryDOS.get(0); + } + return null; + } + + @Override + public int updateChatQuery(ChatQueryDO chatQueryDO) { + return chatQueryDOMapper.updateByPrimaryKeyWithBLOBs(chatQueryDO); + } } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapperTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapperTest.java index 3ebf606a5..217cb9582 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapperTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapperTest.java @@ -11,11 +11,11 @@ class HanlpSchemaMapperTest extends ContextTest { @Test void map() { - QueryContextReq searchCtx = new QueryContextReq(); - searchCtx.setChatId(1); - searchCtx.setDomainId(2); - searchCtx.setQueryText("supersonic按部门访问次数"); + QueryContextReq queryContext = new QueryContextReq(); + queryContext.setChatId(1); + queryContext.setDomainId(2); + queryContext.setQueryText("supersonic按部门访问次数"); HanlpSchemaMapper hanlpSchemaMapper = new HanlpSchemaMapper(); - hanlpSchemaMapper.map(searchCtx); + hanlpSchemaMapper.map(queryContext); } } \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParserTest.java index 7ec42f5e1..73e7ab656 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParserTest.java @@ -13,14 +13,14 @@ class TimeSemanticParserTest { void parse() { TimeSemanticParser timeSemanticParser = new TimeSemanticParser(); - QueryContextReq searchCtx = new QueryContextReq(); + QueryContextReq queryContext = new QueryContextReq(); ChatContext chatCtx = new ChatContext(); SchemaMapInfo schemaMap = new SchemaMapInfo(); - searchCtx.setQueryText("supersonic最近30天访问次数"); + queryContext.setQueryText("supersonic最近30天访问次数"); - boolean parse = timeSemanticParser.parse(searchCtx, chatCtx); + boolean parse = timeSemanticParser.parse(queryContext, chatCtx); - DateConf dateInfo = searchCtx.getParseInfo().getDateInfo(); + DateConf dateInfo = queryContext.getParseInfo().getDateInfo(); } diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java index 31b0e9f50..ad0e0b887 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java @@ -8,7 +8,9 @@ import com.tencent.supersonic.common.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.SchemaItem; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import lombok.Data; public class SemanticParseObjectHelper { @@ -29,9 +31,9 @@ public class SemanticParseObjectHelper { private static SemanticParseInfo getSemanticParseInfo(SemanticParseJson semanticParseJson) { Long domain = semanticParseJson.getDomain(); - List dimensionList = new ArrayList<>(); - List metricList = new ArrayList<>(); - List chatFilters = new ArrayList<>(); + Set dimensionList = new LinkedHashSet(); + Set metricList = new LinkedHashSet(); + Set chatFilters = new LinkedHashSet(); if (semanticParseJson.getFilter() != null && semanticParseJson.getFilter().size() > 0) { for (List filter : semanticParseJson.getFilter()) { diff --git a/chat/knowledge/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java b/chat/knowledge/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java new file mode 100644 index 000000000..47204ec23 --- /dev/null +++ b/chat/knowledge/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java @@ -0,0 +1,337 @@ +package com.hankcs.hanlp.seg; + + +import com.hankcs.hanlp.algorithm.Viterbi; +import com.hankcs.hanlp.collection.AhoCorasick.AhoCorasickDoubleArrayTrie; +import com.hankcs.hanlp.collection.trie.DoubleArrayTrie; +import com.hankcs.hanlp.corpus.tag.Nature; +import com.hankcs.hanlp.dictionary.CoreDictionary; +import com.hankcs.hanlp.dictionary.CoreDictionaryTransformMatrixDictionary; +import com.hankcs.hanlp.dictionary.other.CharType; +import com.hankcs.hanlp.seg.NShort.Path.AtomNode; +import com.hankcs.hanlp.seg.common.Graph; +import com.hankcs.hanlp.seg.common.Term; +import com.hankcs.hanlp.seg.common.Vertex; +import com.hankcs.hanlp.seg.common.WordNet; +import com.hankcs.hanlp.utility.TextUtility; + +import java.util.*; + +public abstract class WordBasedSegment extends Segment { + + public WordBasedSegment() { + } + + protected static void generateWord(List linkedArray, WordNet wordNetOptimum) { + fixResultByRule(linkedArray); + wordNetOptimum.addAll(linkedArray); + } + + protected static void fixResultByRule(List linkedArray) { + mergeContinueNumIntoOne(linkedArray); + changeDelimiterPOS(linkedArray); + splitMiddleSlashFromDigitalWords(linkedArray); + checkDateElements(linkedArray); + } + + static void changeDelimiterPOS(List linkedArray) { + Iterator var1 = linkedArray.iterator(); + + while (true) { + Vertex vertex; + do { + if (!var1.hasNext()) { + return; + } + + vertex = (Vertex) var1.next(); + } while (!vertex.realWord.equals("－－") && !vertex.realWord.equals("—") && !vertex.realWord.equals("-")); + + vertex.confirmNature(Nature.w); + } + } + + private static void splitMiddleSlashFromDigitalWords(List linkedArray) { + if (linkedArray.size() >= 2) { + ListIterator listIterator = linkedArray.listIterator(); + Vertex next = (Vertex) listIterator.next(); + + for (Vertex current = next; listIterator.hasNext(); current = next) { + next = (Vertex) listIterator.next(); + Nature currentNature = current.getNature(); + if (currentNature == Nature.nx && (next.hasNature(Nature.q) || next.hasNature(Nature.n))) { + String[] param = current.realWord.split("-", 1); + if (param.length == 2 && TextUtility.isAllNum(param[0]) && TextUtility.isAllNum(param[1])) { + current = current.copy(); + current.realWord = param[0]; + current.confirmNature(Nature.m); + listIterator.previous(); + listIterator.previous(); + listIterator.set(current); + listIterator.next(); + listIterator.add(Vertex.newPunctuationInstance("-")); + listIterator.add(Vertex.newNumberInstance(param[1])); + } + } + } + + } + } + + private static void checkDateElements(List linkedArray) { + if (linkedArray.size() >= 2) { + ListIterator listIterator = linkedArray.listIterator(); + Vertex next = (Vertex) listIterator.next(); + + for (Vertex current = next; listIterator.hasNext(); current = next) { + next = (Vertex) listIterator.next(); + if (TextUtility.isAllNum(current.realWord) || TextUtility.isAllChineseNum(current.realWord)) { + String nextWord = next.realWord; + if (nextWord.length() == 1 && "月日时分秒".contains(nextWord) + || nextWord.length() == 2 && nextWord.equals("月份")) { + mergeDate(listIterator, next, current); + } else if (nextWord.equals("年")) { + if (TextUtility.isYearTime(current.realWord)) { + mergeDate(listIterator, next, current); + } else { + current.confirmNature(Nature.m); + } + } else if (current.realWord.endsWith("点")) { + current.confirmNature(Nature.t, true); + } else { + char[] tmpCharArray = current.realWord.toCharArray(); + String lastChar = String.valueOf(tmpCharArray[tmpCharArray.length - 1]); + if (!"∶·．／./".contains(lastChar)) { + current.confirmNature(Nature.m, true); + } else if (current.realWord.length() > 1) { + char last = current.realWord.charAt(current.realWord.length() - 1); + current = Vertex.newNumberInstance( + current.realWord.substring(0, current.realWord.length() - 1)); + listIterator.previous(); + listIterator.previous(); + listIterator.set(current); + listIterator.next(); + listIterator.add(Vertex.newPunctuationInstance(String.valueOf(last))); + } + } + } + } + + } + } + + private static void mergeDate(ListIterator listIterator, Vertex next, Vertex current) { + current = Vertex.newTimeInstance(current.realWord + next.realWord); + listIterator.previous(); + listIterator.previous(); + listIterator.set(current); + listIterator.next(); + listIterator.next(); + listIterator.remove(); + } + + protected static List convert(List vertexList) { + return convert(vertexList, false); + } + + protected static Graph generateBiGraph(WordNet wordNet) { + return wordNet.toGraph(); + } + + /** + * @deprecated + */ + private static List atomSegment(String sSentence, int start, int end) { + if (end < start) { + throw new RuntimeException("start=" + start + " < end=" + end); + } else { + List atomSegment = new ArrayList(); + int pCur = 0; + StringBuilder sb = new StringBuilder(); + char[] charArray = sSentence.substring(start, end).toCharArray(); + int[] charTypeArray = new int[charArray.length]; + + for (int i = 0; i < charArray.length; ++i) { + char c = charArray[i]; + charTypeArray[i] = CharType.get(c); + if (c == '.' && i < charArray.length - 1 && CharType.get(charArray[i + 1]) == 9) { + charTypeArray[i] = 9; + } else if (c == '.' && i < charArray.length - 1 && charArray[i + 1] >= '0' && charArray[i + 1] <= '9') { + charTypeArray[i] = 5; + } else if (charTypeArray[i] == 8) { + charTypeArray[i] = 5; + } + } + + while (true) { + while (true) { + while (pCur < charArray.length) { + int nCurType = charTypeArray[pCur]; + if (nCurType != 7 && nCurType != 10 && nCurType != 6 && nCurType != 17) { + if (pCur < charArray.length - 1 && (nCurType == 5 || nCurType == 9)) { + sb.delete(0, sb.length()); + sb.append(charArray[pCur]); + boolean reachEnd = true; + + while (pCur < charArray.length - 1) { + ++pCur; + int nNextType = charTypeArray[pCur]; + if (nNextType != nCurType) { + reachEnd = false; + break; + } + + sb.append(charArray[pCur]); + } + + atomSegment.add(new AtomNode(sb.toString(), nCurType)); + if (reachEnd) { + ++pCur; + } + } else { + atomSegment.add(new AtomNode(charArray[pCur], nCurType)); + ++pCur; + } + } else { + String single = String.valueOf(charArray[pCur]); + if (single.length() != 0) { + atomSegment.add(new AtomNode(single, nCurType)); + } + + ++pCur; + } + } + + return atomSegment; + } + } + } + } + + private static void mergeContinueNumIntoOne(List linkedArray) { + if (linkedArray.size() >= 2) { + ListIterator listIterator = linkedArray.listIterator(); + Vertex next = (Vertex) listIterator.next(); + Vertex current = next; + + while (true) { + while (listIterator.hasNext()) { + next = (Vertex) listIterator.next(); + if (!TextUtility.isAllNum(current.realWord) && !TextUtility.isAllChineseNum(current.realWord) + || !TextUtility.isAllNum(next.realWord) && !TextUtility.isAllChineseNum(next.realWord)) { + current = next; + } else { + current = Vertex.newNumberInstance(current.realWord + next.realWord); + listIterator.previous(); + listIterator.previous(); + listIterator.set(current); + listIterator.next(); + listIterator.next(); + listIterator.remove(); + } + } + + return; + } + } + } + + protected void generateWordNet(final WordNet wordNetStorage) { + final char[] charArray = wordNetStorage.charArray; + DoubleArrayTrie.Searcher searcher = CoreDictionary.trie.getSearcher(charArray, 0); + + while (searcher.next()) { + wordNetStorage.add(searcher.begin + 1, new Vertex(new String(charArray, searcher.begin, searcher.length), + (CoreDictionary.Attribute) searcher.value, searcher.index)); + } + + if (this.config.forceCustomDictionary) { + this.customDictionary.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit() { + public void hit(int begin, int end, CoreDictionary.Attribute value) { + wordNetStorage.add(begin + 1, new Vertex(new String(charArray, begin, end - begin), value)); + } + }); + } + + LinkedList[] vertexes = wordNetStorage.getVertexes(); + int i = 1; + + while (true) { + while (i < vertexes.length) { + if (vertexes[i].isEmpty()) { + int j; + for (j = i + 1; + j < vertexes.length - 1 && (vertexes[j].isEmpty() || CharType.get(charArray[j - 1]) == 11); + ++j) { + } + + wordNetStorage.add(i, quickAtomSegment(charArray, i - 1, j - 1)); + i = j; + } else { + i += ((Vertex) vertexes[i].getLast()).realWord.length(); + } + } + + return; + } + } + + protected List decorateResultForIndexMode(List vertexList, WordNet wordNetAll) { + List termList = new LinkedList(); + int line = 1; + ListIterator listIterator = vertexList.listIterator(); + listIterator.next(); + int length = vertexList.size() - 2; + + for (int i = 0; i < length; ++i) { + Vertex vertex = (Vertex) listIterator.next(); + Term termMain = convert(vertex); + //termList.add(termMain); + addTerms(termList, vertex, line - 1); + termMain.offset = line - 1; + if (vertex.realWord.length() > 2) { + label43: + for (int currentLine = line; currentLine < line + vertex.realWord.length(); ++currentLine) { + Iterator iterator = wordNetAll.descendingIterator(currentLine); + + while (true) { + Vertex smallVertex; + do { + if (!iterator.hasNext()) { + continue label43; + } + smallVertex = (Vertex) iterator.next(); + } while ((termMain.nature != Nature.mq || !smallVertex.hasNature(Nature.q)) + && smallVertex.realWord.length() < this.config.indexMode); + + if (smallVertex != vertex + && currentLine + smallVertex.realWord.length() <= line + vertex.realWord.length()) { + listIterator.add(smallVertex); + //Term termSub = convert(smallVertex); + //termSub.offset = currentLine - 1; + //termList.add(termSub); + addTerms(termList, smallVertex, currentLine - 1); + } + } + } + } + + line += vertex.realWord.length(); + } + + return termList; + } + + protected static void speechTagging(List vertexList) { + Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary); + } + + protected void addTerms(List terms, Vertex vertex, int offset) { + for (int i = 0; i < vertex.attribute.nature.length; i++) { + Term term = new Term(vertex.realWord, vertex.attribute.nature[i]); + term.setFrequency(vertex.attribute.frequency[i]); + term.offset = offset; + terms.add(term); + } + } +} diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/BaseWordNature.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/BaseWordNature.java index 92d4cd769..09873cfd2 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/BaseWordNature.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/BaseWordNature.java @@ -48,4 +48,12 @@ public abstract class BaseWordNature { } return 0; } + + public Long getFrequency(String nature) { + String[] split = nature.split(NatureType.NATURE_SPILT); + if (split.length >= 3) { + return Long.valueOf(split[2]); + } + return 0L; + } } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DimensionWordNature.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DimensionWordNature.java index 41bdc8fd7..7c5bae307 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DimensionWordNature.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DimensionWordNature.java @@ -35,11 +35,11 @@ public class DimensionWordNature extends BaseWordNature { private WordNature getOnwWordNature(String word, ItemDO itemDO, boolean isSuffix) { WordNature wordNature = new WordNature(); wordNature.setWord(word); - Integer classId = itemDO.getDomain(); - String nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId() + Integer domainId = itemDO.getDomain(); + String nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId() + NatureType.DIMENSION.getType(); if (isSuffix) { - nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId() + nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId() + NatureType.SUFFIX.getType() + NatureType.DIMENSION.getType(); } wordNature.setNatureWithFrequency(String.format("%s 100000", nature)); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DomainWordNature.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DomainWordNature.java index e691aa537..8583e64e7 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DomainWordNature.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DomainWordNature.java @@ -20,10 +20,15 @@ public class DomainWordNature extends BaseWordNature { List result = Lists.newArrayList(); WordNature wordNature = new WordNature(); wordNature.setWord(word); - Integer classId = itemDO.getDomain(); - String nature = NatureType.NATURE_SPILT + classId; + Integer domainId = itemDO.getDomain(); + String nature = NatureType.NATURE_SPILT + domainId; wordNature.setNatureWithFrequency(String.format("%s 100000", nature)); result.add(wordNature); return result; } + + @Override + public Long getFrequency(String nature) { + return 0L; + } } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/MetricWordNature.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/MetricWordNature.java index aa0ac190e..34bae35b9 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/MetricWordNature.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/MetricWordNature.java @@ -34,11 +34,11 @@ public class MetricWordNature extends BaseWordNature { private WordNature getOnwWordNature(String word, ItemDO itemDO, boolean isSuffix) { WordNature wordNature = new WordNature(); wordNature.setWord(word); - Integer classId = itemDO.getDomain(); - String nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId() + Integer domainId = itemDO.getDomain(); + String nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId() + NatureType.METRIC.getType(); if (isSuffix) { - nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId() + nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId() + NatureType.SUFFIX.getType() + NatureType.METRIC.getType(); } wordNature.setNatureWithFrequency(String.format("%s 100000", nature)); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/DictionaryAttributeUtil.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/DictionaryAttributeUtil.java index 07821307b..c86cf68dc 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/DictionaryAttributeUtil.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/DictionaryAttributeUtil.java @@ -30,6 +30,9 @@ public class DictionaryAttributeUtil { list.stream().map(i -> i.getKey()).collect(Collectors.toList()).toArray(new Nature[0]), list.stream().map(i -> i.getValue()).mapToInt(Integer::intValue).toArray(), list.stream().map(i -> i.getValue()).findFirst().get()); + if (old.original != null || add.original != null) { + attribute.original = add.original != null ? add.original : old.original; + } return attribute; } } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/HanlpHelper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/HanlpHelper.java index 16bc0a6ef..8b22766f6 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/HanlpHelper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/HanlpHelper.java @@ -3,12 +3,15 @@ package com.tencent.supersonic.knowledge.infrastructure.nlp; import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; import com.hankcs.hanlp.HanLP; +import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.seg.Segment; +import com.tencent.supersonic.common.nlp.MapResult; import com.tencent.supersonic.common.nlp.WordNature; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.ResourceUtils; @@ -153,4 +156,18 @@ public class HanlpHelper { return getDynamicCustomDictionary().insert(wordNature.getWord(), wordNature.getNatureWithFrequency()); } + public static void transLetterOriginal(List mapResults) { + for (MapResult mapResult : mapResults) { + if (MultiCustomDictionary.isLowerLetter(mapResult.getName())) { + if (CustomDictionary.contains(mapResult.getName())) { + CoreDictionary.Attribute attribute = CustomDictionary.get(mapResult.getName()); + if (attribute != null && attribute.original != null) { + mapResult.setName(attribute.original); + } + } + } + } + } + + } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/MultiCustomDictionary.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/MultiCustomDictionary.java index d5b8706db..6c31b5c6b 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/MultiCustomDictionary.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/MultiCustomDictionary.java @@ -359,6 +359,16 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { return false; } + public static boolean isLowerLetter(String str) { + char[] chars = str.toCharArray(); + for (int i = 0; i < chars.length; i++) { + if ((chars[i] >= 'a' && chars[i] <= 'z')) { + return true; + } + } + return false; + } + public static String getWordBySpace(String word) { if (word.contains(HanlpHelper.SPACE_SPILT)) { return word.replace(HanlpHelper.SPACE_SPILT, " "); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/Suggester.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/Suggester.java index 2221ffdfb..1fd7fea87 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/Suggester.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/Suggester.java @@ -53,7 +53,7 @@ public class Suggester { return result.stream().map( entry -> { String name = entry.getKey().replace("#", " "); - return new MapResult(name, entry.getValue()); + return new MapResult(name, entry.getValue(),key); } ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) .limit(SEARCH_SIZE) @@ -79,7 +79,7 @@ public class Suggester { .map(nature -> nature.replaceAll(NatureType.SUFFIX.getType(), "")) .collect(Collectors.toList()); name = StringUtils.reverse(name); - return new MapResult(name, natures); + return new MapResult(name, natures, key); } ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) .limit(SEARCH_SIZE) diff --git a/common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java b/common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java index 9dff3aef1..c4d61caac 100644 --- a/common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java +++ b/common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java @@ -16,13 +16,16 @@ public class MapResult implements Serializable { private double similarity; + private String detectWord; + public MapResult() { } - public MapResult(String name, List natures) { + public MapResult(String name, List natures, String detectWord) { this.name = name; this.natures = natures; + this.detectWord = detectWord; } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java index 116b1be12..4f05f1b30 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Order.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.common.pojo; import static com.tencent.supersonic.common.constant.Constants.ASC_UPPER; +import com.google.common.base.Objects; import javax.validation.constraints.NotBlank; import lombok.Data; @@ -32,4 +33,22 @@ public class Order { sb.append('}'); return sb.toString(); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Order order = (Order) o; + return Objects.equal(column, order.column) && Objects.equal(direction, + order.direction); + } + + @Override + public int hashCode() { + return Objects.hashCode(column, direction); + } } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java b/common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java index 4c2d520f2..7394bf03b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.pojo; +import com.google.common.base.Objects; import com.tencent.supersonic.common.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.enums.StatusEnum; import com.tencent.supersonic.common.enums.TypeEnums; @@ -24,4 +25,26 @@ public class SchemaItem extends RecordInfo { private Integer sensitiveLevel = SensitiveLevelEnum.LOW.getCode(); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + SchemaItem that = (SchemaItem) o; + return Objects.equal(id, that.id) && Objects.equal(name, that.name) + && Objects.equal(bizName, that.bizName) && Objects.equal( + description, that.description) && Objects.equal(status, that.status) + && typeEnum == that.typeEnum && Objects.equal(sensitiveLevel, that.sensitiveLevel); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel); + } } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java b/common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java index f9aab4147..47160c564 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.common.util; +import com.google.common.base.Objects; import java.util.Date; import lombok.Data; import lombok.ToString; @@ -32,4 +33,22 @@ public class RecordInfo { return this; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RecordInfo that = (RecordInfo) o; + return Objects.equal(createdBy, that.createdBy) && Objects.equal( + updatedBy, that.updatedBy) && Objects.equal(createdAt, that.createdAt) + && Objects.equal(updatedAt, that.updatedAt); + } + + @Override + public int hashCode() { + return Objects.hashCode(createdBy, updatedBy, createdAt, updatedAt); + } } diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 038027206..b0497080c 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -1,6 +1,9 @@ com.tencent.supersonic.chat.api.service.SchemaMapper=\ com.tencent.supersonic.chat.application.mapper.HanlpSchemaMapper +com.tencent.supersonic.chat.application.parser.resolver.DomainResolver=\ + com.tencent.supersonic.chat.application.parser.resolver.HeuristicDomainResolver + com.tencent.supersonic.chat.api.service.SemanticParser=\ com.tencent.supersonic.chat.application.parser.TimeSemanticParser, \ com.tencent.supersonic.chat.application.parser.DomainSemanticParser, \ diff --git a/pom.xml b/pom.xml index 111da66b2..a7b34fd6a 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,31 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatabaseServiceImpl.java b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatabaseServiceImpl.java index 1f5d064a4..3c1060008 100644 --- a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatabaseServiceImpl.java +++ b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatabaseServiceImpl.java @@ -93,6 +93,7 @@ public class DatabaseServiceImpl implements DatabaseService { private QueryResultWithSchemaResp queryWithColumns(String sql, DatabaseResp databaseResp) { QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); + log.info("query SQL: {}" , sql); sqlUtils.queryInternal(sql, queryResultWithColumns); return queryResultWithColumns; } diff --git a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/DatasourceRepository.java b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/DatasourceRepository.java index 1e8249601..1286e1583 100644 --- a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/DatasourceRepository.java +++ b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/DatasourceRepository.java @@ -15,7 +15,7 @@ public interface DatasourceRepository { List getDatasourceList(); - List getDatasourceList(Long classId); + List getDatasourceList(Long domainId); DatasourceDO getDatasourceById(Long id); diff --git a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/MetricRepository.java b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/MetricRepository.java index f152d1e2c..31e735d87 100644 --- a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/MetricRepository.java +++ b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/MetricRepository.java @@ -15,7 +15,7 @@ public interface MetricRepository { void updateMetric(MetricDO metricDO); - List getMetricList(Long classId); + List getMetricList(Long domainId); List getMetricListByIds(List ids); diff --git a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DatasourceRepositoryImpl.java b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DatasourceRepositoryImpl.java index eea74cb55..ccf73ad27 100644 --- a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DatasourceRepositoryImpl.java +++ b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DatasourceRepositoryImpl.java @@ -43,9 +43,9 @@ public class DatasourceRepositoryImpl implements DatasourceRepository { } @Override - public List getDatasourceList(Long classId) { + public List getDatasourceList(Long domainId) { DatasourceDOExample datasourceExample = new DatasourceDOExample(); - datasourceExample.createCriteria().andDomainIdEqualTo(classId); + datasourceExample.createCriteria().andDomainIdEqualTo(domainId); return datasourceMapper.selectByExampleWithBLOBs(datasourceExample); } diff --git a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DateInfoRepositoryImpl.java b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DateInfoRepositoryImpl.java index 7c1cb509c..bd4fb6733 100644 --- a/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DateInfoRepositoryImpl.java +++ b/semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DateInfoRepositoryImpl.java @@ -13,9 +13,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; -import javax.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; -import org.apache.ibatis.session.SqlSession; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -25,29 +23,12 @@ import org.springframework.util.CollectionUtils; @Repository public class DateInfoRepositoryImpl implements DateInfoRepository { -// @Autowired -// private SqlSessionTemplate sqlSessionTemplate; - - private SqlSession sqlSession; private ObjectMapper mapper = new ObjectMapper(); @Autowired private DateInfoMapper dateInfoMapper; -// @PostConstruct -// public void init() { -// if (Objects.isNull(sqlSession)) { -// sqlSession = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false); -// } -// } - - @PreDestroy - public void preDestroy() { - if (Objects.nonNull(sqlSession)) { - sqlSession.close(); - } - } @Override public Integer upsertDateInfo(List dateInfoCommends) { @@ -86,12 +67,6 @@ public class DateInfoRepositoryImpl implements DateInfoRepository { for (DateInfoDO dateInfoDO : dateInfoDOList) { dateInfoMapper.upsertDateInfo(dateInfoDO); } - try { - sqlSession.commit(); - return dateInfoDOList.size(); - } catch (Exception e) { - log.warn("e:", e); - } log.info("before final, elapsed time:{}", stopwatch.elapsed(TimeUnit.MILLISECONDS)); return 0; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSchemaManagerImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSchemaManagerImpl.java similarity index 99% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSchemaManagerImpl.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSchemaManagerImpl.java index b6d362f5c..5585d2678 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSchemaManagerImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSchemaManagerImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.application; +package com.tencent.supersonic.semantic.query.application.parser; import com.google.common.cache.CacheBuilder; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSqlServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSqlServiceImpl.java similarity index 95% rename from semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSqlServiceImpl.java rename to semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSqlServiceImpl.java index 7114d1b77..4b35dda26 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSqlServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSqlServiceImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.semantic.query.application; +package com.tencent.supersonic.semantic.query.application.parser; import com.tencent.supersonic.semantic.api.core.response.SqlParserResp; import com.tencent.supersonic.semantic.api.query.request.MetricReq; diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java index 690b7bec8..2cb63855a 100644 --- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java @@ -12,7 +12,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.semantic.query.domain.parser.convertor.planner.AggPlanner; import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema; -import com.tencent.supersonic.semantic.query.application.SemanticSchemaManagerImpl; +import com.tencent.supersonic.semantic.query.application.parser.SemanticSchemaManagerImpl; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap;