topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 02:46:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #4 from lexluo09/master

Browse Source 
[improvement](chat) support query/search filter by web domainId
This commit is contained in:
lexluo09
2023-06-15 18:22:09 +08:00
committed by GitHub
parent c5455c9020 b6f0df40a9
commit 9f55b1c976
57 changed files with 1040 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.Objects;
import lombok.Data;
import lombok.ToString;
@@ -17,4 +18,23 @@ public class Filter {
private Object value;
private Long elementID;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Filter filter = (Filter) o;
return Objects.equals(bizName, filter.bizName) && Objects.equals(name, filter.name)
&& operator == filter.operator && Objects.equals(value, filter.value) && Objects.equals(
elementID, filter.elementID);
}
@Override
public int hashCode() {
return Objects.hash(bizName, name, operator, value, elementID);
}
}

14
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java
View File

@@ -8,17 +8,17 @@ import lombok.ToString;
public class SchemaElementMatch {
SchemaElementType elementType;
int elementID;
double similarity;
String detectWord;
String word;
Long frequency;
public SchemaElementMatch() {
}
public SchemaElementMatch(SchemaElementType schemaElementType, int elementID, double similarity, String word) {
this.elementID = elementID;
this.elementType = schemaElementType;
this.similarity = similarity;
this.word = word;
}
}

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java
View File

@@ -25,4 +25,8 @@ public class SchemaMapInfo {
domainElementMatches.put(domain, elementMatches);
}
public void setDomainElementMatches(
Map<Integer, List<SchemaElementMatch>> domainElementMatches) {
this.domainElementMatches = domainElementMatches;
}
}

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -6,7 +6,9 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import lombok.Data;
@Data
@@ -17,11 +19,11 @@ public class SemanticParseInfo {
Long domainId = 0L;
String domainName;
Long entity = 0L;
List<SchemaItem> metrics = new ArrayList<>();
List<SchemaItem> dimensions = new ArrayList<>();
List<Filter> dimensionFilters = new ArrayList<>();
List<Filter> metricFilters = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
Set<SchemaItem> metrics = new LinkedHashSet();
Set<SchemaItem> dimensions = new LinkedHashSet();
Set<Filter> dimensionFilters = new LinkedHashSet();
Set<Filter> metricFilters = new LinkedHashSet();
private Set<Order> orders = new LinkedHashSet();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/service/SchemaMapper.java
View File

@@ -11,5 +11,5 @@ import com.tencent.supersonic.chat.api.request.QueryContextReq;
*/
public interface SchemaMapper {
void map(QueryContextReq searchCtx);
void map(QueryContextReq queryContext);
}

11
chat/core/src/main/java/com/tencent/supersonic/chat/application/ChatServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.domain.dataobject.ChatDO;
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.domain.dataobject.QueryDO;
import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO;
import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq;
@@ -130,4 +131,14 @@ public class ChatServiceImpl implements ChatService {
chatQueryRepository.createChatQuery(queryResponse, queryContext, chatCtx);
}
@Override
public ChatQueryDO getLastQuery(long chatId) {
return chatQueryRepository.getLastChatQuery(chatId);
}
@Override
public int updateQuery(ChatQueryDO chatQueryDO) {
return chatQueryRepository.updateChatQuery(chatQueryDO);
}
}

12
chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java
View File

@@ -20,6 +20,8 @@ import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.LinkedHashSet;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -137,7 +139,7 @@ public class DomainEntityService {
chatFilter.setValue(String.valueOf(entity));
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(domainInfo));
List<Filter> chatFilters = new ArrayList<>();
Set<Filter> chatFilters = new LinkedHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
@@ -167,8 +169,8 @@ public class DomainEntityService {
}
}
private List<SchemaItem> getDimensions(EntityInfo domainInfo) {
List<SchemaItem> dimensions = new ArrayList<>();
private Set<SchemaItem> getDimensions(EntityInfo domainInfo) {
Set<SchemaItem> dimensions = new LinkedHashSet();
for (DataInfo mainEntityDimension : domainInfo.getDimensions()) {
SchemaItem dimension = new SchemaItem();
dimension.setBizName(mainEntityDimension.getBizName());
@@ -186,8 +188,8 @@ public class DomainEntityService {
return entryKey;
}
private List<SchemaItem> getMetrics(EntityInfo domainInfo) {
List<SchemaItem> metrics = new ArrayList<>();
private Set<SchemaItem> getMetrics(EntityInfo domainInfo) {
Set<SchemaItem> metrics = new LinkedHashSet();
for (DataInfo metricValue : domainInfo.getMetrics()) {
SchemaItem metric = new SchemaItem();
metric.setBizName(metricValue.getBizName());

5
chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java
View File

@@ -3,6 +3,8 @@ package com.tencent.supersonic.chat.application;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
@@ -18,6 +20,9 @@ import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.json.JsonUtil;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;

72
chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java
View File

@@ -4,8 +4,8 @@ import com.google.common.collect.Lists;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.service.SemanticLayer;
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
import com.tencent.supersonic.chat.application.mapper.SearchMatchStrategy;
import com.tencent.supersonic.chat.domain.pojo.search.DomainInfoStat;
import com.tencent.supersonic.chat.domain.pojo.search.DomainWithSemanticType;
@@ -15,12 +15,10 @@ import com.tencent.supersonic.chat.domain.pojo.semantic.DomainInfos;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.domain.service.SearchService;
import com.tencent.supersonic.chat.domain.utils.NatureConverter;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.nlp.ItemDO;
import com.tencent.supersonic.common.nlp.MapResult;
import com.tencent.supersonic.common.nlp.NatureType;
import com.tencent.supersonic.common.nlp.WordNature;
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
import java.util.ArrayList;
import java.util.Comparator;
@@ -47,12 +45,11 @@ import org.springframework.stereotype.Service;
@Service
public class SearchServiceImpl implements SearchService {
private final Logger logger = LoggerFactory.getLogger(SearchServiceImpl.class);
private static final Logger LOGGER = LoggerFactory.getLogger(SearchServiceImpl.class);
@Autowired
private SemanticLayer semanticLayer;
private WordNatureService wordNatureService;
@Autowired
private ChatService chatService;
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@@ -63,13 +60,16 @@ public class SearchServiceImpl implements SearchService {
public List<SearchResult> search(QueryContextReq queryCtx) {
String queryText = queryCtx.getQueryText();
// 1.get meta info
DomainInfos domainInfosDb = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
DomainInfos domainInfosDb = wordNatureService.getCache().getUnchecked("");
List<ItemDO> metricsDb = domainInfosDb.getMetrics();
final Map<Integer, String> domainToName = domainInfosDb.getDomainToName();
// 2.detect by segment
List<Term> originals = HanlpHelper.getSegment().seg(queryText).stream().collect(Collectors.toList());
Map<MatchText, List<MapResult>> regTextMap = searchMatchStrategy.matchWithMatchText(queryText, originals);
List<Term> originals = HanlpHelper.getSegment().seg(queryText.toLowerCase()).stream()
.collect(Collectors.toList());
Map<MatchText, List<MapResult>> regTextMap = searchMatchStrategy.matchWithMatchText(queryText, originals,
queryCtx.getDomainId());
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
// 3.get the most matching data
Optional<Entry<MatchText, List<MapResult>>> mostSimilarSearchResult = regTextMap.entrySet()
.stream()
@@ -77,28 +77,28 @@ public class SearchServiceImpl implements SearchService {
.reduce((entry1, entry2) ->
entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length()
? entry1 : entry2);
logger.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult);
LOGGER.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult);
// 4.optimize the results after the query
if (!mostSimilarSearchResult.isPresent()) {
logger.info("unable to find any information through search , queryCtx:{}", queryCtx);
LOGGER.info("unable to find any information through search , queryCtx:{}", queryCtx);
return Lists.newArrayList();
}
Map.Entry<MatchText, List<MapResult>> searchTextEntry = mostSimilarSearchResult.get();
logger.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx);
LOGGER.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx);
Set<SearchResult> searchResults = new LinkedHashSet();
DomainInfoStat domainStat = NatureHelper.getDomainStat(originals);
List<Integer> possibleDomains = getPossibleDomains(queryCtx, originals, domainStat);
List<Integer> possibleDomains = getPossibleDomains(queryCtx, originals, domainStat, queryCtx.getDomainId());
// 4.1 priority dimension metric
boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDomains), domainToName,
searchTextEntry,
searchResults);
searchTextEntry, searchResults);
// 4.2 process based on dimension values
MatchText matchText = searchTextEntry.getKey();
Map<String, String> natureToNameMap = getNatureToNameMap(searchTextEntry);
Map<String, String> natureToNameMap = getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDomains));
LOGGER.debug("possibleDomains:{},natureToNameMap:{}", possibleDomains, natureToNameMap);
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
searchDimensionValue(metricsDb, domainToName, domainStat.getMetricDomainCount(), searchResults,
@@ -108,12 +108,19 @@ public class SearchServiceImpl implements SearchService {
}
private List<Integer> getPossibleDomains(QueryContextReq queryCtx, List<Term> originals,
DomainInfoStat domainStat) {
DomainInfoStat domainStat, Integer webDomainId) {
if (Objects.nonNull(webDomainId) && webDomainId > 0) {
List<Integer> result = new ArrayList<>();
result.add(webDomainId);
return result;
}
List<Integer> possibleDomains = NatureHelper.selectPossibleDomains(originals);
Long contextDomain = chatService.getContextDomain(queryCtx.getChatId());
logger.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain);
LOGGER.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain);
// If nothing is recognized or only metric are present, then add the contextDomain.
if (nothingOrOnlyMetric(domainStat) && effectiveDomain(contextDomain)) {
@@ -195,16 +202,25 @@ public class SearchServiceImpl implements SearchService {
* @param recommendTextListEntry
* @return
*/
private Map<String, String> getNatureToNameMap(Map.Entry<MatchText, List<MapResult>> recommendTextListEntry) {
private Map<String, String> getNatureToNameMap(Map.Entry<MatchText, List<MapResult>> recommendTextListEntry,
Set<Integer> possibleDomains) {
List<MapResult> recommendValues = recommendTextListEntry.getValue();
return recommendValues.stream()
.flatMap(entry -> entry.getNatures().stream().map(nature -> {
WordNature posDO = new WordNature();
posDO.setWord(entry.getName());
posDO.setNature(nature);
return posDO;
}
)).sorted(Comparator.comparingInt(a -> a.getWord().length()))
.flatMap(entry -> entry.getNatures().stream()
.filter(nature -> {
if (CollectionUtils.isEmpty(possibleDomains)) {
return true;
}
Integer domain = NatureHelper.getDomain(nature);
return possibleDomains.contains(domain);
})
.map(nature -> {
WordNature posDO = new WordNature();
posDO.setWord(entry.getName());
posDO.setNature(nature);
return posDO;
}
)).sorted(Comparator.comparingInt(a -> a.getWord().length()))
.collect(Collectors.toMap(WordNature::getNature, WordNature::getWord, (value1, value2) -> value1,
LinkedHashMap::new));
}
@@ -233,7 +249,7 @@ public class SearchServiceImpl implements SearchService {
domainToName.get(domain), domain, semanticType));
}
}
logger.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult,
LOGGER.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult,
dimensionMetricClassIds, possibleDomains);
}
return existMetric;

9
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java
View File

@@ -10,7 +10,6 @@ import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@@ -23,7 +22,6 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
@Autowired
private WordNatureService wordNatureService;
private List<WordNature> preWordNatures = new ArrayList<>();
@Override
public void onApplicationEvent(ApplicationStartedEvent event) {
@@ -32,7 +30,7 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
this.preWordNatures = wordNatures;
wordNatureService.setPreWordNatures(wordNatures);
onlineKnowledgeService.reloadAllData(wordNatures);
@@ -51,14 +49,15 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
try {
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
List<WordNature> preWordNatures = wordNatureService.getPreWordNatures();
if (CollectionUtils.isEqualCollection(wordNatures, preWordNatures)) {
log.debug("wordNatures is not change, reloadKnowledge end");
return;
}
this.preWordNatures = wordNatures;
wordNatureService.setPreWordNatures(wordNatures);
onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature());
wordNatureService.getCache().refresh("");
} catch (Exception e) {
log.error("reloadKnowledge error", e);

3
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java
View File

@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
public class NatureHelper {
private static final Logger LOGGER = LoggerFactory.getLogger(NatureHelper.class);
private static boolean isDomainOrEntity(Term term, Integer domain) {
return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString()
.endsWith(NatureType.ENTITY.getType());
@@ -96,7 +97,7 @@ public class NatureHelper {
/**
* Get the number of types of class parts of speech
* classId -> (nature , natureCount)
* domainId -> (nature , natureCount)
*
* @param terms
* @return

35
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.chat.application.knowledge;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.chat.api.service.SemanticLayer;
import com.tencent.supersonic.chat.domain.pojo.semantic.DomainInfos;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
@@ -9,6 +12,7 @@ import com.tencent.supersonic.common.nlp.WordNature;
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -21,10 +25,25 @@ import org.springframework.stereotype.Service;
@Service
public class WordNatureService {
private final Logger logger = LoggerFactory.getLogger(WordNatureService.class);
private static final Logger LOGGER = LoggerFactory.getLogger(WordNatureService.class);
@Autowired
private SemanticLayer semanticLayer;
private static final Integer META_CACHE_TIME = 5;
private List<WordNature> preWordNatures = new ArrayList<>();
private LoadingCache<String, DomainInfos> cache = CacheBuilder.newBuilder()
.expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
.build(
new CacheLoader<String, DomainInfos>() {
@Override
public DomainInfos load(String key) {
LOGGER.info("load getDomainSchemaInfo cache [{}]", key);
return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
}
}
);
public List<WordNature> getAllWordNature() {
@@ -45,7 +64,19 @@ public class WordNatureService {
private void addNatureToResult(NatureType value, List<ItemDO> metas, List<WordNature> natures) {
List<WordNature> natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas);
logger.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
LOGGER.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
natures.addAll(natureList);
}
public List<WordNature> getPreWordNatures() {
return preWordNatures;
}
public void setPreWordNatures(List<WordNature> preWordNatures) {
this.preWordNatures = preWordNatures;
}
public LoadingCache<String, DomainInfos> getCache() {
return cache;
}
}

24
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java
View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.nlp.MapResult;
import com.tencent.supersonic.common.nlp.NatureType;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
import java.util.ArrayList;
@@ -28,18 +29,22 @@ public class HanlpSchemaMapper implements SchemaMapper {
private static final Logger LOGGER = LoggerFactory.getLogger(HanlpSchemaMapper.class);
@Override
public void map(QueryContextReq searchCtx) {
public void map(QueryContextReq queryContext) {
List<Term> terms = HanlpHelper.getSegment().seg(queryContext.getQueryText().toLowerCase()).stream()
.collect(Collectors.toList());
List<Term> terms = HanlpHelper.getSegment().seg(searchCtx.getQueryText()).stream().collect(Collectors.toList());
terms.forEach(
item -> LOGGER.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(), item.frequency)
item -> LOGGER.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
item.getFrequency())
);
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
List<MapResult> matches = matchStrategy.match(searchCtx.getQueryText(), terms, 0);
LOGGER.info("searchCtx:{},matches:{}", searchCtx, matches);
List<MapResult> matches = matchStrategy.match(queryContext.getQueryText(), terms, queryContext.getDomainId());
HanlpHelper.transLetterOriginal(matches);
LOGGER.info("queryContext:{},matches:{}", queryContext, matches);
convertTermsToSchemaMapInfo(matches, searchCtx.getMapInfo());
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo());
}
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap) {
@@ -59,11 +64,14 @@ public class HanlpSchemaMapper implements SchemaMapper {
SchemaElementMatch schemaElementMatch = new SchemaElementMatch();
schemaElementMatch.setElementType(elementType);
Integer elementID = WordNatureStrategyFactory.get(NatureType.getNatureType(nature))
.getElementID(nature);
BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature));
Integer elementID = baseWordNature.getElementID(nature);
schemaElementMatch.setElementID(elementID);
Long frequency = baseWordNature.getFrequency(nature);
schemaElementMatch.setFrequency(frequency);
schemaElementMatch.setWord(mapResult.getName());
schemaElementMatch.setSimilarity(mapResult.getSimilarity());
schemaElementMatch.setDetectWord(mapResult.getDetectWord());
Map<Integer, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domain,

12
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/MatchStrategy.java
View File

@@ -13,18 +13,10 @@ import java.util.Map;
*/
public interface MatchStrategy {
/***
* match
* @param terms
* @return
*/
List<MapResult> match(String text, List<Term> terms, int retryCount);
List<MapResult> match(String text, List<Term> terms, Integer detectDomainId);
List<MapResult> match(String text, List<Term> terms, int retryCount, Integer detectDomainId);
Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals);
Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals, Integer detectDomainId);
/***
* exist dimension values

36
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryMatchStrategy.java
View File

@@ -40,35 +40,31 @@ public class QueryMatchStrategy implements MatchStrategy {
private Double dimensionValueThresholdConfig;
@Override
public List<MapResult> match(String text, List<Term> terms, int retryCount) {
return match(text, terms, retryCount, null);
}
@Override
public List<MapResult> match(String text, List<Term> terms, int retryCount, Integer detectDomainId) {
public List<MapResult> match(String text, List<Term> terms, Integer detectDomainId) {
if (CollectionUtils.isEmpty(terms) || StringUtils.isEmpty(text)) {
return null;
}
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
List<Integer> offsetList = terms.stream().sorted(Comparator.comparing(Term::getOffset))
.map(term -> term.getOffset()).collect(Collectors.toList());
LOGGER.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", retryCount, terms,
regOffsetToLength, offsetList,
detectDomainId);
LOGGER.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", terms,
regOffsetToLength, offsetList, detectDomainId);
return detect(text, regOffsetToLength, offsetList, detectDomainId, retryCount);
return detect(text, regOffsetToLength, offsetList, detectDomainId);
}
@Override
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals) {
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals,
Integer detectDomainId) {
return null;
}
private List<MapResult> detect(String text, Map<Integer, Integer> regOffsetToLength, List<Integer> offsetList,
Integer detectDomainId, int retryCount) {
Integer detectDomainId) {
List<MapResult> results = Lists.newArrayList();
for (Integer index = 0; index <= text.length() - 1; ) {
@@ -79,7 +75,7 @@ public class QueryMatchStrategy implements MatchStrategy {
int offset = getStepOffset(offsetList, index);
i = getStepIndex(regOffsetToLength, i);
if (i <= text.length()) {
List<MapResult> mapResults = detectByStep(text, detectDomainId, index, i, offset, retryCount);
List<MapResult> mapResults = detectByStep(text, detectDomainId, index, i, offset);
mapResultRowSet.addAll(mapResults);
}
}
@@ -90,8 +86,7 @@ public class QueryMatchStrategy implements MatchStrategy {
return results;
}
private List<MapResult> detectByStep(String text, Integer detectClassId, Integer index, Integer i, int offset,
int retryCount) {
private List<MapResult> detectByStep(String text, Integer detectDomainId, Integer index, Integer i, int offset) {
String detectSegment = text.substring(index, i);
// step1. pre search
LinkedHashSet<MapResult> mapResults = Suggester.prefixSearch(detectSegment, oneDetectionMaxSize)
@@ -109,11 +104,11 @@ public class QueryMatchStrategy implements MatchStrategy {
mapResults = mapResults.stream().sorted((a, b) -> -(b.getName().length() - a.getName().length()))
.collect(Collectors.toCollection(LinkedHashSet::new));
// step4. filter by classId
if (Objects.nonNull(detectClassId) && detectClassId > 0) {
if (Objects.nonNull(detectDomainId) && detectDomainId > 0) {
LOGGER.debug("detectDomainId:{}, before parseResults:{}", mapResults);
mapResults = mapResults.stream().map(entry -> {
List<String> natures = entry.getNatures().stream().filter(
nature -> nature.startsWith(NatureType.NATURE_SPILT + detectClassId) || (nature.startsWith(
nature -> nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) || (nature.startsWith(
NatureType.NATURE_SPILT))
).collect(Collectors.toList());
entry.setNatures(natures);
@@ -123,8 +118,7 @@ public class QueryMatchStrategy implements MatchStrategy {
}
// step5. filter by similarity
mapResults = mapResults.stream()
.filter(term -> getSimilarity(detectSegment, term.getName()) >= getThresholdMatch(term.getNatures(),
retryCount))
.filter(term -> getSimilarity(detectSegment, term.getName()) >= getThresholdMatch(term.getNatures()))
.filter(term -> CollectionUtils.isNotEmpty(term.getNatures()))
.collect(Collectors.toCollection(LinkedHashSet::new));
@@ -170,11 +164,11 @@ public class QueryMatchStrategy implements MatchStrategy {
return index;
}
private double getThresholdMatch(List<String> natures, int retryCount) {
private double getThresholdMatch(List<String> natures) {
if (existDimensionValues(natures)) {
return dimensionValueThresholdConfig;
}
return metricDimensionThresholdConfig - STEP * retryCount;
return metricDimensionThresholdConfig;
}
}

24
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/SearchMatchStrategy.java
View File

@@ -23,20 +23,16 @@ public class SearchMatchStrategy implements MatchStrategy {
private static final int SEARCH_SIZE = 3;
@Override
public List<MapResult> match(String text, List<Term> terms, int retryCount) {
public List<MapResult> match(String text, List<Term> terms, Integer detectDomainId) {
return null;
}
@Override
public List<MapResult> match(String text, List<Term> terms, int retryCount, Integer detectDomainId) {
return null;
}
@Override
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals) {
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals,
Integer detectDomainId) {
Map<Integer, Integer> regOffsetToLength = originals.stream()
.filter(entry -> !entry.nature.toString().startsWith(NatureType.NATURE_SPILT))
@@ -70,7 +66,17 @@ public class SearchMatchStrategy implements MatchStrategy {
mapResults = mapResults.stream().filter(entry -> {
List<String> natures = entry.getNatures().stream()
.filter(nature -> !nature.endsWith(NatureType.ENTITY.getType()))
.collect(Collectors.toList());
.filter(nature -> {
if (Objects.isNull(detectDomainId) || detectDomainId <= 0) {
return true;
}
if (nature.startsWith(NatureType.NATURE_SPILT + detectDomainId)
&& nature.startsWith(NatureType.NATURE_SPILT)) {
return true;
}
return false;
}
).collect(Collectors.toList());
if (CollectionUtils.isEmpty(natures)) {
return false;
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java
View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.context.ContextUtils;
import java.util.List;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -21,7 +22,7 @@ import org.springframework.stereotype.Component;
public class AggregateSemanticParser implements SemanticParser {
private final Logger logger = LoggerFactory.getLogger(AggregateSemanticParser.class);
public static final Integer TOPN_LIMIT = 10;
public static final Integer TOPN_LIMIT = 1000;
private AggregateTypeResolver aggregateTypeResolver;
@@ -33,7 +34,7 @@ public class AggregateSemanticParser implements SemanticParser {
SemanticParseInfo semanticParse = queryContext.getParseInfo();
List<SchemaItem> metrics = semanticParse.getMetrics();
Set<SchemaItem> metrics = semanticParse.getMetrics();
semanticParse.setNativeQuery(getNativeQuery(aggregateType, queryContext));
@@ -47,12 +48,12 @@ public class AggregateSemanticParser implements SemanticParser {
/**
* query mode reset by the AggregateType
*
* @param searchCtx
* @param queryContext
* @param aggregateType
*/
private void resetQueryModeByAggregateType(QueryContextReq searchCtx, AggregateTypeEnum aggregateType) {
private void resetQueryModeByAggregateType(QueryContextReq queryContext, AggregateTypeEnum aggregateType) {
SemanticParseInfo parseInfo = searchCtx.getParseInfo();
SemanticParseInfo parseInfo = queryContext.getParseInfo();
String queryMode = parseInfo.getQueryMode();
if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) {
if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType)
@@ -63,7 +64,7 @@ public class AggregateSemanticParser implements SemanticParser {
}
}
if (MetricFilter.QUERY_MODE.equals(queryMode) || MetricCompare.QUERY_MODE.equals(queryMode)) {
if (aggregateTypeResolver.hasCompareIntentionalWords(searchCtx.getQueryText())) {
if (aggregateTypeResolver.hasCompareIntentionalWords(queryContext.getQueryText())) {
parseInfo.setQueryMode(MetricCompare.QUERY_MODE);
} else {
parseInfo.setQueryMode(MetricFilter.QUERY_MODE);
@@ -72,11 +73,11 @@ public class AggregateSemanticParser implements SemanticParser {
logger.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode());
}
private boolean getNativeQuery(AggregateTypeEnum aggregateType, QueryContextReq searchCtx) {
private boolean getNativeQuery(AggregateTypeEnum aggregateType, QueryContextReq queryContext) {
if (AggregateTypeEnum.TOPN.equals(aggregateType)) {
return true;
}
return searchCtx.getParseInfo().getNativeQuery();
return queryContext.getParseInfo().getNativeQuery();
}

60
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DefaultMetricSemanticParser.java
View File

@@ -9,8 +9,6 @@ import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.service.SemanticParser;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import com.tencent.supersonic.chat.application.parser.resolver.DomainResolver;
import com.tencent.supersonic.chat.application.query.EntityDetail;
import com.tencent.supersonic.chat.application.query.EntityListFilter;
@@ -24,8 +22,10 @@ import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.context.ContextUtils;
import java.util.ArrayList;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@@ -104,33 +104,39 @@ public class DefaultMetricSemanticParser implements SemanticParser {
return primaryDimensions;
}
protected void addEntityDetailDimensionMetric(QueryContextReq searchCtx, ChatContext chatCtx) {
if (searchCtx.getParseInfo().getDomainId() > 0) {
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
searchCtx.getParseInfo().getDomainId());
protected void addEntityDetailDimensionMetric(QueryContextReq queryContext, ChatContext chatCtx) {
if (queryContext.getParseInfo().getDomainId() > 0) {
Long domainId = queryContext.getParseInfo().getDomainId();
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domainId);
if (chaConfigRichDesc != null) {
SemanticParseInfo semanticParseInfo = searchCtx.getParseInfo();
if (Objects.nonNull(semanticParseInfo) && CollectionUtils.isEmpty(semanticParseInfo.getDimensions())) {
List<SchemaItem> dimensions = new ArrayList<>();
List<SchemaItem> metrics = new ArrayList<>();
if (chaConfigRichDesc.getEntity() != null
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
.forEach(m -> metrics.add(getMetric(m)));
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
.forEach(m -> dimensions.add(getDimension(m)));
}
semanticParseInfo.setDimensions(dimensions);
semanticParseInfo.setMetrics(metrics);
if (chaConfigRichDesc.getEntity() == null
|| chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() == null) {
return;
}
SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
Set<SchemaItem> metrics = new LinkedHashSet();
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
.forEach(m -> metrics.add(getMetric(m)));
semanticParseInfo.setMetrics(metrics);
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
.getMatchedElements(domainId.intValue());
if (CollectionUtils.isEmpty(schemaElementMatches) || schemaElementMatches.stream()
.filter(s -> SchemaElementType.DIMENSION.equals(s.getElementType())).count() <= 0) {
logger.info("addEntityDetailDimensionMetric catch");
Set<SchemaItem> dimensions = new LinkedHashSet();
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
.forEach(m -> dimensions.add(getDimension(m)));
semanticParseInfo.setDimensions(dimensions);
}
}
}
}
protected void defaultQueryMode(QueryContextReq searchCtx, ChatContext chatCtx) {
SchemaMapInfo schemaMap = searchCtx.getMapInfo();
SemanticParseInfo parseInfo = searchCtx.getParseInfo();
protected void defaultQueryMode(QueryContextReq queryContext, ChatContext chatCtx) {
SchemaMapInfo schemaMap = queryContext.getMapInfo();
SemanticParseInfo parseInfo = queryContext.getParseInfo();
if (StringUtils.isEmpty(parseInfo.getQueryMode())) {
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) {
//
@@ -182,12 +188,12 @@ public class DefaultMetricSemanticParser implements SemanticParser {
}
private void fillDateDomain(ChatContext chatCtx, QueryContextReq searchCtx) {
SemanticParseInfo parseInfo = searchCtx.getParseInfo();
private void fillDateDomain(ChatContext chatCtx, QueryContextReq queryContext) {
SemanticParseInfo parseInfo = queryContext.getParseInfo();
if (parseInfo == null || parseInfo.getDateInfo() == null) {
boolean isUpdateTime = false;
if (selectStrategy.isDomainSwitch(chatCtx, searchCtx)) {
if (selectStrategy.isDomainSwitch(chatCtx, queryContext)) {
isUpdateTime = true;
}
if (chatCtx.getParseInfo() == null
@@ -212,7 +218,7 @@ public class DefaultMetricSemanticParser implements SemanticParser {
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) && CollectionUtils.isEmpty(
semanticParseInfo.getDimensions())) {
List<SchemaItem> metrics = new ArrayList<>();
Set<SchemaItem> metrics = new LinkedHashSet();
chaConfigRichDesc.getDefaultMetrics().stream().forEach(metric -> {
SchemaItem metricTmp = new SchemaItem();
metricTmp.setId(metric.getMetricId());

32
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java
View File

@@ -25,6 +25,7 @@ import java.util.Objects;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.support.SpringFactoriesLoader;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -32,10 +33,15 @@ import org.springframework.util.CollectionUtils;
public class DomainSemanticParser implements SemanticParser {
private final Logger logger = LoggerFactory.getLogger(DomainSemanticParser.class);
private DomainResolver domainResolver;
private List<DomainResolver> domainResolverList;
private SemanticQueryResolver semanticQueryResolver;
public DomainSemanticParser() {
domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class,
Thread.currentThread().getContextClassLoader());
}
@Override
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
DomainInfos domainInfosDb = SchemaInfoConverter.convert(
@@ -45,7 +51,7 @@ public class DomainSemanticParser implements SemanticParser {
SchemaMapInfo mapInfo = queryContext.getMapInfo();
SemanticParseInfo parseInfo = queryContext.getParseInfo();
domainResolver = ContextUtils.getBean(DomainResolver.class);
//domainResolver = ContextUtils.getBean(DomainResolver.class);
semanticQueryResolver = ContextUtils.getBean(SemanticQueryResolver.class);
Map<Integer, SemanticQuery> domainSemanticQuery = new HashMap<>();
@@ -72,15 +78,19 @@ public class DomainSemanticParser implements SemanticParser {
}
} else if (domainSemanticQuery.size() > 1) {
// will choose one by the domain select
Integer domainId = domainResolver.resolve(domainSemanticQuery, queryContext, chatCtx, mapInfo);
if (domainId > 0) {
Map.Entry<Integer, SemanticQuery> match = domainSemanticQuery.entrySet().stream()
.filter(entry -> entry.getKey().equals(domainId)).findFirst().orElse(null);
logger.info("select by selectStrategy [{}:{}]", domainId, match.getValue());
parseInfo.setDomainId(Long.valueOf(match.getKey()));
parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.getKey())));
parseInfo.setQueryMode(match.getValue().getQueryMode());
return false;
Optional<Integer> domainId = domainResolverList.stream()
.map(domainResolver -> domainResolver.resolve(domainSemanticQuery, queryContext, chatCtx, mapInfo))
.filter(d -> d > 0).findFirst();
if (domainId.isPresent() && domainId.get() > 0) {
for (Map.Entry<Integer, SemanticQuery> match : domainSemanticQuery.entrySet()) {
if (match.getKey().equals(domainId.get())) {
logger.info("select by selectStrategy [{}:{}]", domainId.get(), match.getValue());
parseInfo.setDomainId(Long.valueOf(match.getKey()));
parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.getKey())));
parseInfo.setQueryMode(match.getValue().getQueryMode());
return false;
}
}
}
}
// Round 2: no domains can be found yet, count in chat context

25
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/ListFilterParser.java
View File

@@ -19,6 +19,7 @@ import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.context.ContextUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@@ -43,7 +44,7 @@ public class ListFilterParser implements SemanticParser {
this.fillDateEntityFilter(queryContext.getParseInfo());
this.addEntityDetailAndOrderByMetric(queryContext, chatCtx);
this.dealNativeQuery(queryContext, true);
return false;
return true;
}
@@ -56,15 +57,15 @@ public class ListFilterParser implements SemanticParser {
semanticParseInfo.setDateInfo(dateInfo);
}
private void addEntityDetailAndOrderByMetric(QueryContextReq searchCtx, ChatContext chatCtx) {
if (searchCtx.getParseInfo().getDomainId() > 0L) {
private void addEntityDetailAndOrderByMetric(QueryContextReq queryContext, ChatContext chatCtx) {
if (queryContext.getParseInfo().getDomainId() > 0L) {
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
searchCtx.getParseInfo().getDomainId());
queryContext.getParseInfo().getDomainId());
if (chaConfigRichDesc != null) {
SemanticParseInfo semanticParseInfo = searchCtx.getParseInfo();
List<SchemaItem> dimensions = new ArrayList();
SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
Set<SchemaItem> dimensions = new LinkedHashSet();
Set<String> primaryDimensions = this.addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions);
List<SchemaItem> metrics = new ArrayList();
Set<SchemaItem> metrics = new LinkedHashSet();
if (chaConfigRichDesc.getEntity() != null
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
@@ -76,7 +77,7 @@ public class ListFilterParser implements SemanticParser {
semanticParseInfo.setDimensions(dimensions);
semanticParseInfo.setMetrics(metrics);
List<Order> orders = new ArrayList();
Set<Order> orders = new LinkedHashSet();
if (chaConfigRichDesc.getEntity() != null
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
@@ -89,7 +90,7 @@ public class ListFilterParser implements SemanticParser {
}
private Set<String> addPrimaryDimension(EntityRichInfo entity, List<SchemaItem> dimensions) {
private Set<String> addPrimaryDimension(EntityRichInfo entity, Set<SchemaItem> dimensions) {
Set<String> primaryDimensions = new HashSet();
if (!Objects.isNull(entity) && !CollectionUtils.isEmpty(entity.getEntityIds())) {
entity.getEntityIds().stream().forEach((dimSchemaDesc) -> {
@@ -120,9 +121,9 @@ public class ListFilterParser implements SemanticParser {
return queryMeta;
}
private void dealNativeQuery(QueryContextReq searchCtx, boolean isNativeQuery) {
if (Objects.nonNull(searchCtx) && Objects.nonNull(searchCtx.getParseInfo())) {
searchCtx.getParseInfo().setNativeQuery(isNativeQuery);
private void dealNativeQuery(QueryContextReq queryContext, boolean isNativeQuery) {
if (Objects.nonNull(queryContext) && Objects.nonNull(queryContext.getParseInfo())) {
queryContext.getParseInfo().setNativeQuery(isNativeQuery);
}
}

131
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/BaseDomainResolver.java Normal file
View File

@@ -0,0 +1,131 @@
package com.tencent.supersonic.chat.application.parser.resolver;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.service.SemanticQuery;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class BaseDomainResolver implements DomainResolver {
@Override
public boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq searchCtx) {
Long contextDomain = chatCtx.getParseInfo().getDomainId();
Long currentDomain = searchCtx.getParseInfo().getDomainId();
boolean noSwitch = currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
log.info("ChatContext isDomainSwitch [{}]", !noSwitch);
return !noSwitch;
}
public abstract Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx, SchemaMapInfo schemaMap);
@Override
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx, SchemaMapInfo schemaMap) {
Integer selectDomain = selectDomain(domainQueryModes, searchCtx, chatCtx, schemaMap);
if (selectDomain > 0) {
log.info("selectDomain {} ", selectDomain);
return selectDomain;
}
// get the max SchemaElementType number
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
}
protected static Integer selectDomainBySchemaElementCount(Map<Integer, SemanticQuery> domainQueryModes,
SchemaMapInfo schemaMap) {
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
if (domainTypeMap.size() == 1) {
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
if (domainQueryModes.containsKey(domainSelect)) {
log.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
return domainSelect;
}
} else {
Map.Entry<Integer, SchemaElementCount> maxDomain = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
if (maxDomain != null) {
log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey());
return maxDomain.getKey();
}
}
return 0;
}
/**
* to check can switch domain if context exit domain
*
* @return false will use context domain, true will use other domain , maybe include context domain
*/
protected static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
if (!Objects.nonNull(domainId) || domainId <= 0) {
return true;
}
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
.filter(entry -> entry.getValue().getCount() > 1).count();
if (otherDomainTypeNumBigOneCount >= 1) {
return true;
}
// if query text only contain time , will not switch
if (searchCtx.getQueryText() != null && searchCtx.getParseInfo().getDateInfo() != null) {
if (searchCtx.getParseInfo().getDateInfo().getText() != null) {
if (searchCtx.getParseInfo().getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
log.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
searchCtx.getParseInfo().getDateInfo());
return false;
}
}
}
// if context domain not in schemaMap , will switch
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
log.info("domainId not in schemaMap ");
return true;
}
// other will not switch
return false;
}
protected static Map<Integer, SchemaElementCount> getDomainTypeMap(SchemaMapInfo schemaMap) {
Map<Integer, SchemaElementCount> domainCount = new HashMap<>();
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
if (!domainCount.containsKey(entry.getKey())) {
domainCount.put(entry.getKey(), new SchemaElementCount());
}
SchemaElementCount schemaElementCount = domainCount.get(entry.getKey());
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
schemaElementMatches.stream()
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
if (schemaElementMatchMax != null) {
schemaElementCount.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
}
schemaElementCount.setCount(schemaElementTypes.size());
}
}
return domainCount;
}
}

112
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/HeuristicDomainResolver.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.service.SemanticQuery;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Primary;
@@ -22,121 +23,28 @@ import java.util.Set;
import java.util.stream.Collectors;
@Service("heuristicDomainResolver")
@Primary
public class HeuristicDomainResolver implements DomainResolver {
private static final Logger LOGGER = LoggerFactory.getLogger(HeuristicDomainResolver.class);
@Service("DomainResolver")
@Slf4j
public class HeuristicDomainResolver extends BaseDomainResolver {
@Override
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx, SchemaMapInfo schemaMap) {
public Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx,
SchemaMapInfo schemaMap) {
// if QueryContext has domainId and in domainQueryModes
if (domainQueryModes.containsKey(searchCtx.getDomainId())) {
LOGGER.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
return searchCtx.getDomainId();
}
// if ChatContext has domainId and in domainQueryModes
if (chatCtx.getParseInfo().getDomainId() > 0) {
Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) {
LOGGER.info("selectDomain from ChatContext [{}]", domainId);
log.info("selectDomain from ChatContext [{}]", domainId);
return domainId;
}
}
// get the max SchemaElementType number
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
if (domainTypeMap.size() == 1) {
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
if (domainQueryModes.containsKey(domainSelect)) {
LOGGER.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
return domainSelect;
}
} else {
Map.Entry<Integer, SchemaElementCount> maxDomain = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
if (maxDomain != null) {
LOGGER.info("selectDomain from domainTypeMap need order [{}]", maxDomain.getKey());
return maxDomain.getKey();
}
}
// bad case , here will not reach , default 0
LOGGER.error("selectDomain not found ");
// default 0
return 0;
}
@Override
public boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq searchCtx) {
Long contextDomain = chatCtx.getParseInfo().getDomainId();
Long currentDomain = searchCtx.getParseInfo().getDomainId();
boolean noSwitch = currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
LOGGER.info("ChatContext isDomainSwitch [{}]", !noSwitch);
return !noSwitch;
}
/**
* to check can switch domain if context exit domain
*
* @return false will use context domain, true will use other domain , maybe include context domain
*/
private static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
if (!Objects.nonNull(domainId) || domainId <= 0) {
return true;
}
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
LOGGER.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
.filter(entry -> entry.getValue().getCount() > 1).count();
if (otherDomainTypeNumBigOneCount >= 1) {
return true;
}
// if query text only contain time , will not switch
if (searchCtx.getQueryText() != null && searchCtx.getParseInfo().getDateInfo() != null) {
if (searchCtx.getParseInfo().getDateInfo().getText() != null) {
if (searchCtx.getParseInfo().getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
LOGGER.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
searchCtx.getParseInfo().getDateInfo());
return false;
}
}
}
// if context domain not in schemaMap , will switch
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
LOGGER.info("domainId not in schemaMap ");
return true;
}
// other will not switch
return false;
}
private static Map<Integer, SchemaElementCount> getDomainTypeMap(SchemaMapInfo schemaMap) {
Map<Integer, SchemaElementCount> domainCount = new HashMap<>();
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
if (!domainCount.containsKey(entry.getKey())) {
domainCount.put(entry.getKey(), new SchemaElementCount());
}
SchemaElementCount schemaElementCount = domainCount.get(entry.getKey());
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
schemaElementMatches.stream()
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
if (schemaElementMatchMax != null) {
schemaElementCount.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
}
schemaElementCount.setCount(schemaElementTypes.size());
}
}
return domainCount;
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/resolver/RegexAggregateTypeResolver.java
View File

@@ -22,7 +22,7 @@ public class RegexAggregateTypeResolver implements AggregateTypeResolver {
aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高)"));
aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低)"));
aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)"));
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|平均|avg)"));
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)"));
aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)"));
aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)"));
aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)"));

2
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java
View File

@@ -11,6 +11,7 @@ import org.springframework.stereotype.Service;
public class EntityListFilter extends BaseSemanticQuery {
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
private static Long entityListLimit = 200L;
public EntityListFilter() {
queryModeOption.setAggregation(QueryModeElementOption.unused());
@@ -42,6 +43,7 @@ public class EntityListFilter extends BaseSemanticQuery {
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
semanticParseInfo.setLimit(entityListLimit);
return semanticParseInfo;
}

67
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java
View File

@@ -1,13 +1,24 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.Filter;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class MetricCompare extends BaseSemanticQuery {
public static String QUERY_MODE = "METRIC_COMPARE";
@@ -48,7 +59,61 @@ public class MetricCompare extends BaseSemanticQuery {
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
ContextHelper.appendList(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
mergeAppend(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
addCompareDimension(semanticParseInfo);
return semanticParseInfo;
}
private void addCompareDimension(SemanticParseInfo semanticParseInfo) {
if (!semanticParseInfo.getDimensionFilters().isEmpty()) {
Set<String> dimensions = semanticParseInfo.getDimensions().stream().map(d -> d.getBizName()).collect(
Collectors.toSet());
log.info("addCompareDimension before [{}]", dimensions);
semanticParseInfo.getDimensionFilters().stream().filter(d -> d.getOperator().equals(FilterOperatorEnum.IN))
.forEach(
d -> {
if (!dimensions.contains(d.getBizName())) {
SchemaItem schemaItem = new SchemaItem();
schemaItem.setBizName(d.getBizName());
schemaItem.setId(d.getElementID());
semanticParseInfo.getDimensions().add(schemaItem);
dimensions.add(d.getBizName());
}
}
);
log.info("addCompareDimension after [{}]", dimensions);
}
}
private void mergeAppend(Set<Filter> from, Set<Filter> to) {
if (!from.isEmpty()) {
for (Filter filter : from) {
if (FilterOperatorEnum.EQUALS.equals(filter.getOperator()) || FilterOperatorEnum.IN.equals(
filter.getOperator())) {
Optional<Filter> toAdd = to.stream()
.filter(t -> t.getBizName().equalsIgnoreCase(filter.getBizName())).findFirst();
if (toAdd.isPresent()) {
if (FilterOperatorEnum.EQUALS.equals(toAdd.get().getOperator()) || FilterOperatorEnum.IN.equals(
toAdd.get().getOperator())) {
List<Object> vals = new ArrayList<>();
if (toAdd.get().getOperator().equals(FilterOperatorEnum.IN)) {
vals.addAll((List<Object>) (toAdd.get().getValue()));
} else {
vals.add(toAdd.get().getValue());
}
if (filter.getOperator().equals(FilterOperatorEnum.IN)) {
vals.addAll((List<Object>) (filter.getValue()));
} else {
vals.add(filter.getValue());
}
toAdd.get().setValue(vals);
toAdd.get().setOperator(FilterOperatorEnum.IN);
continue;
}
}
}
to.add(filter);
}
}
}
}

16
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java
View File

@@ -2,21 +2,25 @@ package com.tencent.supersonic.chat.domain.pojo.chat;
import com.tencent.supersonic.chat.api.pojo.Filter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class QueryData {
Long domainId = 0L;
List<SchemaItem> metrics = new ArrayList<>();
List<SchemaItem> dimensions = new ArrayList<>();
List<Filter> filters = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
Set<SchemaItem> metrics = new HashSet<>();
Set<SchemaItem> dimensions = new HashSet<>();
Set<Filter> dimensionFilters = new HashSet<>();
Set<Filter> metricFilters = new HashSet<>();
private Set<Order> orders = new HashSet<>();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;

4
chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java
View File

@@ -4,6 +4,7 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO;
import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq;
@@ -13,4 +14,7 @@ public interface ChatQueryRepository {
void createChatQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx);
ChatQueryDO getLastChatQuery(long chatId);
int updateChatQuery(ChatQueryDO chatQueryDO);
}

5
chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ChatService.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.domain.dataobject.ChatDO;
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO;
import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq;
import java.util.List;
@@ -40,4 +41,8 @@ public interface ChatService {
PageInfo<ChatQueryVO> queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId);
public void addQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx);
public ChatQueryDO getLastQuery(long chatId);
public int updateQuery(ChatQueryDO chatQueryDO);
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/ContextHelper.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.api.service.SemanticQuery;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -71,7 +72,7 @@ public class ContextHelper {
* @param from
* @param to
*/
public static void addIfEmpty(List from, List to) {
public static void addIfEmpty(Set from, Set to) {
if (to.isEmpty() && !from.isEmpty()) {
to.addAll(from);
}
@@ -82,7 +83,7 @@ public class ContextHelper {
* @param from
* @param to
*/
public static void appendList(List from, List to) {
public static void appendList(Set from, Set to) {
if (!from.isEmpty()) {
to.addAll(from);
}
@@ -94,7 +95,7 @@ public class ContextHelper {
* @param from
* @param to
*/
public static void updateList(List from, List to) {
public static void updateList(Set from, Set to) {
if (!from.isEmpty()) {
to.clear();
to.addAll(from);

15
chat/core/src/main/java/com/tencent/supersonic/chat/domain/utils/SchemaInfoConverter.java
View File

@@ -21,9 +21,14 @@ import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils;
public class SchemaInfoConverter {
@@ -40,6 +45,7 @@ public class SchemaInfoConverter {
queryStructCmd.setDateInfo(parseInfo.getDateInfo());
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
.collect(Collectors.toList());
queryStructCmd.setDimensionFilters(dimensionFilters);
@@ -54,12 +60,13 @@ public class SchemaInfoConverter {
.collect(Collectors.toList());
queryStructCmd.setGroups(dimensions);
queryStructCmd.setLimit(parseInfo.getLimit());
queryStructCmd.setOrders(getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics()));
Set<Order> order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics());
queryStructCmd.setOrders(new ArrayList<>(order));
queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getMetrics(), parseInfo.getAggType()));
return queryStructCmd;
}
private static List<Aggregator> getAggregatorByMetric(List<SchemaItem> metrics, AggregateTypeEnum aggregateType) {
private static List<Aggregator> getAggregatorByMetric(Set<SchemaItem> metrics, AggregateTypeEnum aggregateType) {
List<Aggregator> aggregators = new ArrayList<>();
String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? ""
: aggregateType.name();
@@ -157,11 +164,11 @@ public class SchemaInfoConverter {
return result;
}
public static List<Order> getOrder(List<Order> parseOrder, AggregateTypeEnum aggregator, List<SchemaItem> metrics) {
public static Set<Order> getOrder(Set<Order> parseOrder, AggregateTypeEnum aggregator, Set<SchemaItem> metrics) {
if (!CollectionUtils.isEmpty(parseOrder)) {
return parseOrder;
}
List<Order> orders = new ArrayList<>();
Set<Order> orders = new LinkedHashSet();
if (CollectionUtils.isEmpty(metrics)) {
return orders;
}

22
chat/core/src/main/java/com/tencent/supersonic/chat/infrastructure/repository/ChatQueryRepositoryImpl.java
View File

@@ -14,11 +14,13 @@ import com.tencent.supersonic.chat.domain.repository.ChatQueryRepository;
import com.tencent.supersonic.chat.infrastructure.mapper.ChatQueryDOMapper;
import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.common.util.mybatis.PageUtils;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Repository;
import org.springframework.util.CollectionUtils;
@Repository
@Primary
@@ -69,4 +71,24 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
Long queryId = Long.valueOf(chatQueryDOMapper.insert(chatQueryDO));
queryResponse.setQueryId(queryId);
}
@Override
public ChatQueryDO getLastChatQuery(long chatId) {
ChatQueryDOExample example = new ChatQueryDOExample();
example.setOrderByClause("question_id desc");
example.setLimitEnd(1);
example.setLimitStart(0);
Criteria criteria = example.createCriteria();
criteria.andChatIdEqualTo(chatId);
List<ChatQueryDO> chatQueryDOS = chatQueryDOMapper.selectByExampleWithBLOBs(example);
if (!CollectionUtils.isEmpty(chatQueryDOS)) {
return chatQueryDOS.get(0);
}
return null;
}
@Override
public int updateChatQuery(ChatQueryDO chatQueryDO) {
return chatQueryDOMapper.updateByPrimaryKeyWithBLOBs(chatQueryDO);
}
}

10
chat/core/src/test/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapperTest.java
View File

@@ -11,11 +11,11 @@ class HanlpSchemaMapperTest extends ContextTest {
@Test
void map() {
QueryContextReq searchCtx = new QueryContextReq();
searchCtx.setChatId(1);
searchCtx.setDomainId(2);
searchCtx.setQueryText("supersonic按部门访问次数");
QueryContextReq queryContext = new QueryContextReq();
queryContext.setChatId(1);
queryContext.setDomainId(2);
queryContext.setQueryText("supersonic按部门访问次数");
HanlpSchemaMapper hanlpSchemaMapper = new HanlpSchemaMapper();
hanlpSchemaMapper.map(searchCtx);
hanlpSchemaMapper.map(queryContext);
}
}

8
chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParserTest.java
View File

@@ -13,14 +13,14 @@ class TimeSemanticParserTest {
void parse() {
TimeSemanticParser timeSemanticParser = new TimeSemanticParser();
QueryContextReq searchCtx = new QueryContextReq();
QueryContextReq queryContext = new QueryContextReq();
ChatContext chatCtx = new ChatContext();
SchemaMapInfo schemaMap = new SchemaMapInfo();
searchCtx.setQueryText("supersonic最近30天访问次数");
queryContext.setQueryText("supersonic最近30天访问次数");
boolean parse = timeSemanticParser.parse(searchCtx, chatCtx);
boolean parse = timeSemanticParser.parse(queryContext, chatCtx);
DateConf dateInfo = searchCtx.getParseInfo().getDateInfo();
DateConf dateInfo = queryContext.getParseInfo().getDateInfo();
}

8
chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java
View File

@@ -8,7 +8,9 @@ import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import lombok.Data;
public class SemanticParseObjectHelper {
@@ -29,9 +31,9 @@ public class SemanticParseObjectHelper {
private static SemanticParseInfo getSemanticParseInfo(SemanticParseJson semanticParseJson) {
Long domain = semanticParseJson.getDomain();
List<SchemaItem> dimensionList = new ArrayList<>();
List<SchemaItem> metricList = new ArrayList<>();
List<Filter> chatFilters = new ArrayList<>();
Set<SchemaItem> dimensionList = new LinkedHashSet();
Set<SchemaItem> metricList = new LinkedHashSet();
Set<Filter> chatFilters = new LinkedHashSet();
if (semanticParseJson.getFilter() != null && semanticParseJson.getFilter().size() > 0) {
for (List<String> filter : semanticParseJson.getFilter()) {

337
chat/knowledge/src/main/java/com/hankcs/hanlp/seg/WordBasedSegment.java Normal file
View File

@@ -0,0 +1,337 @@
package com.hankcs.hanlp.seg;
import com.hankcs.hanlp.algorithm.Viterbi;
import com.hankcs.hanlp.collection.AhoCorasick.AhoCorasickDoubleArrayTrie;
import com.hankcs.hanlp.collection.trie.DoubleArrayTrie;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.dictionary.CoreDictionary;
import com.hankcs.hanlp.dictionary.CoreDictionaryTransformMatrixDictionary;
import com.hankcs.hanlp.dictionary.other.CharType;
import com.hankcs.hanlp.seg.NShort.Path.AtomNode;
import com.hankcs.hanlp.seg.common.Graph;
import com.hankcs.hanlp.seg.common.Term;
import com.hankcs.hanlp.seg.common.Vertex;
import com.hankcs.hanlp.seg.common.WordNet;
import com.hankcs.hanlp.utility.TextUtility;
import java.util.*;
public abstract class WordBasedSegment extends Segment {
public WordBasedSegment() {
}
protected static void generateWord(List<Vertex> linkedArray, WordNet wordNetOptimum) {
fixResultByRule(linkedArray);
wordNetOptimum.addAll(linkedArray);
}
protected static void fixResultByRule(List<Vertex> linkedArray) {
mergeContinueNumIntoOne(linkedArray);
changeDelimiterPOS(linkedArray);
splitMiddleSlashFromDigitalWords(linkedArray);
checkDateElements(linkedArray);
}
static void changeDelimiterPOS(List<Vertex> linkedArray) {
Iterator var1 = linkedArray.iterator();
while (true) {
Vertex vertex;
do {
if (!var1.hasNext()) {
return;
}
vertex = (Vertex) var1.next();
} while (!vertex.realWord.equals("－－") && !vertex.realWord.equals("—") && !vertex.realWord.equals("-"));
vertex.confirmNature(Nature.w);
}
}
private static void splitMiddleSlashFromDigitalWords(List<Vertex> linkedArray) {
if (linkedArray.size() >= 2) {
ListIterator<Vertex> listIterator = linkedArray.listIterator();
Vertex next = (Vertex) listIterator.next();
for (Vertex current = next; listIterator.hasNext(); current = next) {
next = (Vertex) listIterator.next();
Nature currentNature = current.getNature();
if (currentNature == Nature.nx && (next.hasNature(Nature.q) || next.hasNature(Nature.n))) {
String[] param = current.realWord.split("-", 1);
if (param.length == 2 && TextUtility.isAllNum(param[0]) && TextUtility.isAllNum(param[1])) {
current = current.copy();
current.realWord = param[0];
current.confirmNature(Nature.m);
listIterator.previous();
listIterator.previous();
listIterator.set(current);
listIterator.next();
listIterator.add(Vertex.newPunctuationInstance("-"));
listIterator.add(Vertex.newNumberInstance(param[1]));
}
}
}
}
}
private static void checkDateElements(List<Vertex> linkedArray) {
if (linkedArray.size() >= 2) {
ListIterator<Vertex> listIterator = linkedArray.listIterator();
Vertex next = (Vertex) listIterator.next();
for (Vertex current = next; listIterator.hasNext(); current = next) {
next = (Vertex) listIterator.next();
if (TextUtility.isAllNum(current.realWord) || TextUtility.isAllChineseNum(current.realWord)) {
String nextWord = next.realWord;
if (nextWord.length() == 1 && "月日时分秒".contains(nextWord)
|| nextWord.length() == 2 && nextWord.equals("月份")) {
mergeDate(listIterator, next, current);
} else if (nextWord.equals("年")) {
if (TextUtility.isYearTime(current.realWord)) {
mergeDate(listIterator, next, current);
} else {
current.confirmNature(Nature.m);
}
} else if (current.realWord.endsWith("点")) {
current.confirmNature(Nature.t, true);
} else {
char[] tmpCharArray = current.realWord.toCharArray();
String lastChar = String.valueOf(tmpCharArray[tmpCharArray.length - 1]);
if (!"∶·．／./".contains(lastChar)) {
current.confirmNature(Nature.m, true);
} else if (current.realWord.length() > 1) {
char last = current.realWord.charAt(current.realWord.length() - 1);
current = Vertex.newNumberInstance(
current.realWord.substring(0, current.realWord.length() - 1));
listIterator.previous();
listIterator.previous();
listIterator.set(current);
listIterator.next();
listIterator.add(Vertex.newPunctuationInstance(String.valueOf(last)));
}
}
}
}
}
}
private static void mergeDate(ListIterator<Vertex> listIterator, Vertex next, Vertex current) {
current = Vertex.newTimeInstance(current.realWord + next.realWord);
listIterator.previous();
listIterator.previous();
listIterator.set(current);
listIterator.next();
listIterator.next();
listIterator.remove();
}
protected static List<Term> convert(List<Vertex> vertexList) {
return convert(vertexList, false);
}
protected static Graph generateBiGraph(WordNet wordNet) {
return wordNet.toGraph();
}
/**
* @deprecated
*/
private static List<AtomNode> atomSegment(String sSentence, int start, int end) {
if (end < start) {
throw new RuntimeException("start=" + start + " < end=" + end);
} else {
List<AtomNode> atomSegment = new ArrayList();
int pCur = 0;
StringBuilder sb = new StringBuilder();
char[] charArray = sSentence.substring(start, end).toCharArray();
int[] charTypeArray = new int[charArray.length];
for (int i = 0; i < charArray.length; ++i) {
char c = charArray[i];
charTypeArray[i] = CharType.get(c);
if (c == '.' && i < charArray.length - 1 && CharType.get(charArray[i + 1]) == 9) {
charTypeArray[i] = 9;
} else if (c == '.' && i < charArray.length - 1 && charArray[i + 1] >= '0' && charArray[i + 1] <= '9') {
charTypeArray[i] = 5;
} else if (charTypeArray[i] == 8) {
charTypeArray[i] = 5;
}
}
while (true) {
while (true) {
while (pCur < charArray.length) {
int nCurType = charTypeArray[pCur];
if (nCurType != 7 && nCurType != 10 && nCurType != 6 && nCurType != 17) {
if (pCur < charArray.length - 1 && (nCurType == 5 || nCurType == 9)) {
sb.delete(0, sb.length());
sb.append(charArray[pCur]);
boolean reachEnd = true;
while (pCur < charArray.length - 1) {
++pCur;
int nNextType = charTypeArray[pCur];
if (nNextType != nCurType) {
reachEnd = false;
break;
}
sb.append(charArray[pCur]);
}
atomSegment.add(new AtomNode(sb.toString(), nCurType));
if (reachEnd) {
++pCur;
}
} else {
atomSegment.add(new AtomNode(charArray[pCur], nCurType));
++pCur;
}
} else {
String single = String.valueOf(charArray[pCur]);
if (single.length() != 0) {
atomSegment.add(new AtomNode(single, nCurType));
}
++pCur;
}
}
return atomSegment;
}
}
}
}
private static void mergeContinueNumIntoOne(List<Vertex> linkedArray) {
if (linkedArray.size() >= 2) {
ListIterator<Vertex> listIterator = linkedArray.listIterator();
Vertex next = (Vertex) listIterator.next();
Vertex current = next;
while (true) {
while (listIterator.hasNext()) {
next = (Vertex) listIterator.next();
if (!TextUtility.isAllNum(current.realWord) && !TextUtility.isAllChineseNum(current.realWord)
|| !TextUtility.isAllNum(next.realWord) && !TextUtility.isAllChineseNum(next.realWord)) {
current = next;
} else {
current = Vertex.newNumberInstance(current.realWord + next.realWord);
listIterator.previous();
listIterator.previous();
listIterator.set(current);
listIterator.next();
listIterator.next();
listIterator.remove();
}
}
return;
}
}
}
protected void generateWordNet(final WordNet wordNetStorage) {
final char[] charArray = wordNetStorage.charArray;
DoubleArrayTrie.Searcher searcher = CoreDictionary.trie.getSearcher(charArray, 0);
while (searcher.next()) {
wordNetStorage.add(searcher.begin + 1, new Vertex(new String(charArray, searcher.begin, searcher.length),
(CoreDictionary.Attribute) searcher.value, searcher.index));
}
if (this.config.forceCustomDictionary) {
this.customDictionary.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit<CoreDictionary.Attribute>() {
public void hit(int begin, int end, CoreDictionary.Attribute value) {
wordNetStorage.add(begin + 1, new Vertex(new String(charArray, begin, end - begin), value));
}
});
}
LinkedList<Vertex>[] vertexes = wordNetStorage.getVertexes();
int i = 1;
while (true) {
while (i < vertexes.length) {
if (vertexes[i].isEmpty()) {
int j;
for (j = i + 1;
j < vertexes.length - 1 && (vertexes[j].isEmpty() || CharType.get(charArray[j - 1]) == 11);
++j) {
}
wordNetStorage.add(i, quickAtomSegment(charArray, i - 1, j - 1));
i = j;
} else {
i += ((Vertex) vertexes[i].getLast()).realWord.length();
}
}
return;
}
}
protected List<Term> decorateResultForIndexMode(List<Vertex> vertexList, WordNet wordNetAll) {
List<Term> termList = new LinkedList();
int line = 1;
ListIterator<Vertex> listIterator = vertexList.listIterator();
listIterator.next();
int length = vertexList.size() - 2;
for (int i = 0; i < length; ++i) {
Vertex vertex = (Vertex) listIterator.next();
Term termMain = convert(vertex);
//termList.add(termMain);
addTerms(termList, vertex, line - 1);
termMain.offset = line - 1;
if (vertex.realWord.length() > 2) {
label43:
for (int currentLine = line; currentLine < line + vertex.realWord.length(); ++currentLine) {
Iterator iterator = wordNetAll.descendingIterator(currentLine);
while (true) {
Vertex smallVertex;
do {
if (!iterator.hasNext()) {
continue label43;
}
smallVertex = (Vertex) iterator.next();
} while ((termMain.nature != Nature.mq || !smallVertex.hasNature(Nature.q))
&& smallVertex.realWord.length() < this.config.indexMode);
if (smallVertex != vertex
&& currentLine + smallVertex.realWord.length() <= line + vertex.realWord.length()) {
listIterator.add(smallVertex);
//Term termSub = convert(smallVertex);
//termSub.offset = currentLine - 1;
//termList.add(termSub);
addTerms(termList, smallVertex, currentLine - 1);
}
}
}
}
line += vertex.realWord.length();
}
return termList;
}
protected static void speechTagging(List<Vertex> vertexList) {
Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
}
protected void addTerms(List<Term> terms, Vertex vertex, int offset) {
for (int i = 0; i < vertex.attribute.nature.length; i++) {
Term term = new Term(vertex.realWord, vertex.attribute.nature[i]);
term.setFrequency(vertex.attribute.frequency[i]);
term.offset = offset;
terms.add(term);
}
}
}

8
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/BaseWordNature.java
View File

@@ -48,4 +48,12 @@ public abstract class BaseWordNature {
}
return 0;
}
public Long getFrequency(String nature) {
String[] split = nature.split(NatureType.NATURE_SPILT);
if (split.length >= 3) {
return Long.valueOf(split[2]);
}
return 0L;
}
}

6
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DimensionWordNature.java
View File

@@ -35,11 +35,11 @@ public class DimensionWordNature extends BaseWordNature {
private WordNature getOnwWordNature(String word, ItemDO itemDO, boolean isSuffix) {
WordNature wordNature = new WordNature();
wordNature.setWord(word);
Integer classId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId()
Integer domainId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId()
+ NatureType.DIMENSION.getType();
if (isSuffix) {
nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId()
nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId()
+ NatureType.SUFFIX.getType() + NatureType.DIMENSION.getType();
}
wordNature.setNatureWithFrequency(String.format("%s 100000", nature));

9
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/DomainWordNature.java
View File

@@ -20,10 +20,15 @@ public class DomainWordNature extends BaseWordNature {
List<WordNature> result = Lists.newArrayList();
WordNature wordNature = new WordNature();
wordNature.setWord(word);
Integer classId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + classId;
Integer domainId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + domainId;
wordNature.setNatureWithFrequency(String.format("%s 100000", nature));
result.add(wordNature);
return result;
}
@Override
public Long getFrequency(String nature) {
return 0L;
}
}

6
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/application/online/MetricWordNature.java
View File

@@ -34,11 +34,11 @@ public class MetricWordNature extends BaseWordNature {
private WordNature getOnwWordNature(String word, ItemDO itemDO, boolean isSuffix) {
WordNature wordNature = new WordNature();
wordNature.setWord(word);
Integer classId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId()
Integer domainId = itemDO.getDomain();
String nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId()
+ NatureType.METRIC.getType();
if (isSuffix) {
nature = NatureType.NATURE_SPILT + classId + NatureType.NATURE_SPILT + itemDO.getItemId()
nature = NatureType.NATURE_SPILT + domainId + NatureType.NATURE_SPILT + itemDO.getItemId()
+ NatureType.SUFFIX.getType() + NatureType.METRIC.getType();
}
wordNature.setNatureWithFrequency(String.format("%s 100000", nature));

3
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/DictionaryAttributeUtil.java
View File

@@ -30,6 +30,9 @@ public class DictionaryAttributeUtil {
list.stream().map(i -> i.getKey()).collect(Collectors.toList()).toArray(new Nature[0]),
list.stream().map(i -> i.getValue()).mapToInt(Integer::intValue).toArray(),
list.stream().map(i -> i.getValue()).findFirst().get());
if (old.original != null || add.original != null) {
attribute.original = add.original != null ? add.original : old.original;
}
return attribute;
}
}

17
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/HanlpHelper.java
View File

@@ -3,12 +3,15 @@ package com.tencent.supersonic.knowledge.infrastructure.nlp;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.CoreDictionary;
import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
import com.hankcs.hanlp.seg.Segment;
import com.tencent.supersonic.common.nlp.MapResult;
import com.tencent.supersonic.common.nlp.WordNature;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
@@ -153,4 +156,18 @@ public class HanlpHelper {
return getDynamicCustomDictionary().insert(wordNature.getWord(), wordNature.getNatureWithFrequency());
}
public static void transLetterOriginal(List<MapResult> mapResults) {
for (MapResult mapResult : mapResults) {
if (MultiCustomDictionary.isLowerLetter(mapResult.getName())) {
if (CustomDictionary.contains(mapResult.getName())) {
CoreDictionary.Attribute attribute = CustomDictionary.get(mapResult.getName());
if (attribute != null && attribute.original != null) {
mapResult.setName(attribute.original);
}
}
}
}
}
}

10
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/MultiCustomDictionary.java
View File

@@ -359,6 +359,16 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
return false;
}
public static boolean isLowerLetter(String str) {
char[] chars = str.toCharArray();
for (int i = 0; i < chars.length; i++) {
if ((chars[i] >= 'a' && chars[i] <= 'z')) {
return true;
}
}
return false;
}
public static String getWordBySpace(String word) {
if (word.contains(HanlpHelper.SPACE_SPILT)) {
return word.replace(HanlpHelper.SPACE_SPILT, " ");

4
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/infrastructure/nlp/Suggester.java
View File

@@ -53,7 +53,7 @@ public class Suggester {
return result.stream().map(
entry -> {
String name = entry.getKey().replace("#", " ");
return new MapResult(name, entry.getValue());
return new MapResult(name, entry.getValue(),key);
}
).sorted((a, b) -> -(b.getName().length() - a.getName().length()))
.limit(SEARCH_SIZE)
@@ -79,7 +79,7 @@ public class Suggester {
.map(nature -> nature.replaceAll(NatureType.SUFFIX.getType(), ""))
.collect(Collectors.toList());
name = StringUtils.reverse(name);
return new MapResult(name, natures);
return new MapResult(name, natures, key);
}
).sorted((a, b) -> -(b.getName().length() - a.getName().length()))
.limit(SEARCH_SIZE)

5
common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java
View File

@@ -16,13 +16,16 @@ public class MapResult implements Serializable {
private double similarity;
private String detectWord;
public MapResult() {
}
public MapResult(String name, List<String> natures) {
public MapResult(String name, List<String> natures, String detectWord) {
this.name = name;
this.natures = natures;
this.detectWord = detectWord;
}
@Override

19
common/src/main/java/com/tencent/supersonic/common/pojo/Order.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.common.pojo;
import static com.tencent.supersonic.common.constant.Constants.ASC_UPPER;
import com.google.common.base.Objects;
import javax.validation.constraints.NotBlank;
import lombok.Data;
@@ -32,4 +33,22 @@ public class Order {
sb.append('}');
return sb.toString();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Order order = (Order) o;
return Objects.equal(column, order.column) && Objects.equal(direction,
order.direction);
}
@Override
public int hashCode() {
return Objects.hashCode(column, direction);
}
}

23
common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.google.common.base.Objects;
import com.tencent.supersonic.common.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.enums.StatusEnum;
import com.tencent.supersonic.common.enums.TypeEnums;
@@ -24,4 +25,26 @@ public class SchemaItem extends RecordInfo {
private Integer sensitiveLevel = SensitiveLevelEnum.LOW.getCode();
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
SchemaItem that = (SchemaItem) o;
return Objects.equal(id, that.id) && Objects.equal(name, that.name)
&& Objects.equal(bizName, that.bizName) && Objects.equal(
description, that.description) && Objects.equal(status, that.status)
&& typeEnum == that.typeEnum && Objects.equal(sensitiveLevel, that.sensitiveLevel);
}
@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel);
}
}

19
common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.util;
import com.google.common.base.Objects;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
@@ -32,4 +33,22 @@ public class RecordInfo {
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
RecordInfo that = (RecordInfo) o;
return Objects.equal(createdBy, that.createdBy) && Objects.equal(
updatedBy, that.updatedBy) && Objects.equal(createdAt, that.createdAt)
&& Objects.equal(updatedAt, that.updatedAt);
}
@Override
public int hashCode() {
return Objects.hashCode(createdBy, updatedBy, createdAt, updatedAt);
}
}

3
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -1,6 +1,9 @@
com.tencent.supersonic.chat.api.service.SchemaMapper=\
com.tencent.supersonic.chat.application.mapper.HanlpSchemaMapper
com.tencent.supersonic.chat.application.parser.resolver.DomainResolver=\
com.tencent.supersonic.chat.application.parser.resolver.HeuristicDomainResolver
com.tencent.supersonic.chat.api.service.SemanticParser=\
com.tencent.supersonic.chat.application.parser.TimeSemanticParser, \
com.tencent.supersonic.chat.application.parser.DomainSemanticParser, \

25
pom.xml
View File

@@ -100,6 +100,31 @@
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>

1
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatabaseServiceImpl.java
View File

@@ -93,6 +93,7 @@ public class DatabaseServiceImpl implements DatabaseService {
private QueryResultWithSchemaResp queryWithColumns(String sql, DatabaseResp databaseResp) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);
log.info("query SQL: {}" , sql);
sqlUtils.queryInternal(sql, queryResultWithColumns);
return queryResultWithColumns;
}

2
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/DatasourceRepository.java
View File

@@ -15,7 +15,7 @@ public interface DatasourceRepository {
List<DatasourceDO> getDatasourceList();
List<DatasourceDO> getDatasourceList(Long classId);
List<DatasourceDO> getDatasourceList(Long domainId);
DatasourceDO getDatasourceById(Long id);

2
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/repository/MetricRepository.java
View File

@@ -15,7 +15,7 @@ public interface MetricRepository {
void updateMetric(MetricDO metricDO);
List<MetricDO> getMetricList(Long classId);
List<MetricDO> getMetricList(Long domainId);
List<MetricDO> getMetricListByIds(List<Long> ids);

4
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DatasourceRepositoryImpl.java
View File

@@ -43,9 +43,9 @@ public class DatasourceRepositoryImpl implements DatasourceRepository {
}
@Override
public List<DatasourceDO> getDatasourceList(Long classId) {
public List<DatasourceDO> getDatasourceList(Long domainId) {
DatasourceDOExample datasourceExample = new DatasourceDOExample();
datasourceExample.createCriteria().andDomainIdEqualTo(classId);
datasourceExample.createCriteria().andDomainIdEqualTo(domainId);
return datasourceMapper.selectByExampleWithBLOBs(datasourceExample);
}

25
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/DateInfoRepositoryImpl.java
View File

@@ -13,9 +13,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import javax.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.session.SqlSession;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
@@ -25,29 +23,12 @@ import org.springframework.util.CollectionUtils;
@Repository
public class DateInfoRepositoryImpl implements DateInfoRepository {
// @Autowired
// private SqlSessionTemplate sqlSessionTemplate;
private SqlSession sqlSession;
private ObjectMapper mapper = new ObjectMapper();
@Autowired
private DateInfoMapper dateInfoMapper;
// @PostConstruct
// public void init() {
// if (Objects.isNull(sqlSession)) {
// sqlSession = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);
// }
// }
@PreDestroy
public void preDestroy() {
if (Objects.nonNull(sqlSession)) {
sqlSession.close();
}
}
@Override
public Integer upsertDateInfo(List<DateInfoReq> dateInfoCommends) {
@@ -86,12 +67,6 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
for (DateInfoDO dateInfoDO : dateInfoDOList) {
dateInfoMapper.upsertDateInfo(dateInfoDO);
}
try {
sqlSession.commit();
return dateInfoDOList.size();
} catch (Exception e) {
log.warn("e:", e);
}
log.info("before final, elapsed time:{}", stopwatch.elapsed(TimeUnit.MILLISECONDS));
return 0;
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSchemaManagerImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSchemaManagerImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application;
package com.tencent.supersonic.semantic.query.application.parser;
import com.google.common.cache.CacheBuilder;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticSqlServiceImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSqlServiceImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application;
package com.tencent.supersonic.semantic.query.application.parser;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;

2
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.SemanticSchemaManagerImpl;
import com.tencent.supersonic.semantic.query.application.parser.SemanticSchemaManagerImpl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;