mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-14 13:47:09 +00:00
[improvement][project] supersonic 0.7.0 version backend update (#20)
Co-authored-by: kanedai <kanedai@tencent.com>
This commit is contained in:
daikon
@@ -1,213 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application;
|
||||
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.pojo.DataInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.DomainInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DomainEntityService {
|
||||
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
@Autowired
|
||||
private ConfigServiceImpl configService;
|
||||
|
||||
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
|
||||
if (parseInfo != null && parseInfo.getDomainId() > 0) {
|
||||
EntityInfo entityInfo = getEntityInfo(parseInfo.getDomainId());
|
||||
if (parseInfo.getDimensionFilters().size() <= 0) {
|
||||
entityInfo.setMetrics(null);
|
||||
entityInfo.setDimensions(null);
|
||||
return entityInfo;
|
||||
}
|
||||
if (entityInfo.getDomainInfo() != null && entityInfo.getDomainInfo().getPrimaryEntityBizName() != null) {
|
||||
String domainInfoPrimaryName = entityInfo.getDomainInfo().getPrimaryEntityBizName();
|
||||
String domainInfoId = "";
|
||||
for (Filter chatFilter : parseInfo.getDimensionFilters()) {
|
||||
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
|
||||
.equals(domainInfoPrimaryName)) {
|
||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
||||
domainInfoId = chatFilter.getValue().toString();
|
||||
}
|
||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.IN)) {
|
||||
domainInfoId = ((List<String>) chatFilter.getValue()).get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!"".equals(domainInfoId)) {
|
||||
try {
|
||||
setMainDomain(entityInfo, parseInfo.getDomainId(),
|
||||
domainInfoId, user);
|
||||
|
||||
return entityInfo;
|
||||
} catch (Exception e) {
|
||||
log.error("setMaintDomain error {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public EntityInfo getEntityInfo(Long domain) {
|
||||
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain);
|
||||
if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) {
|
||||
return new EntityInfo();
|
||||
}
|
||||
return getEntityInfo(chaConfigRichDesc);
|
||||
}
|
||||
|
||||
private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) {
|
||||
|
||||
EntityInfo entityInfo = new EntityInfo();
|
||||
EntityRichInfo entityDesc = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
|
||||
if (entityDesc != null && Objects.nonNull(chaConfigRichDesc.getDomainId())) {
|
||||
DomainInfo domainInfo = new DomainInfo();
|
||||
domainInfo.setItemId(Integer.valueOf(chaConfigRichDesc.getDomainId().intValue()));
|
||||
domainInfo.setName(chaConfigRichDesc.getDomainName());
|
||||
domainInfo.setWords(entityDesc.getNames());
|
||||
domainInfo.setBizName(chaConfigRichDesc.getBizName());
|
||||
if (Objects.nonNull(entityDesc.getDimItem())) {
|
||||
domainInfo.setPrimaryEntityBizName(entityDesc.getDimItem().getBizName());
|
||||
}
|
||||
|
||||
entityInfo.setDomainInfo(domainInfo);
|
||||
List<DataInfo> dimensions = new ArrayList<>();
|
||||
List<DataInfo> metrics = new ArrayList<>();
|
||||
|
||||
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())
|
||||
&& Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) {
|
||||
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
|
||||
if(!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())){
|
||||
for (SchemaItem dimensionDesc : chatDefaultConfig.getDimensions()) {
|
||||
DataInfo mainEntityDimension = new DataInfo();
|
||||
mainEntityDimension.setItemId(dimensionDesc.getId().intValue());
|
||||
mainEntityDimension.setName(dimensionDesc.getName());
|
||||
mainEntityDimension.setBizName(dimensionDesc.getBizName());
|
||||
dimensions.add(mainEntityDimension);
|
||||
}
|
||||
entityInfo.setDimensions(dimensions);
|
||||
}
|
||||
|
||||
if(!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())){
|
||||
for (SchemaItem metricDesc : chatDefaultConfig.getMetrics()) {
|
||||
DataInfo dataInfo = new DataInfo();
|
||||
dataInfo.setName(metricDesc.getName());
|
||||
dataInfo.setBizName(metricDesc.getBizName());
|
||||
dataInfo.setItemId(metricDesc.getId().intValue());
|
||||
metrics.add(dataInfo);
|
||||
}
|
||||
entityInfo.setMetrics(metrics);
|
||||
}
|
||||
}
|
||||
}
|
||||
return entityInfo;
|
||||
}
|
||||
|
||||
public void setMainDomain(EntityInfo domainInfo, Long domain, String entity, User user) {
|
||||
domainInfo.setEntityId(entity);
|
||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||
semanticParseInfo.setDomainId(Long.valueOf(domain));
|
||||
semanticParseInfo.setNativeQuery(true);
|
||||
semanticParseInfo.setMetrics(getMetrics(domainInfo));
|
||||
semanticParseInfo.setDimensions(getDimensions(domainInfo));
|
||||
DateConf dateInfo = new DateConf();
|
||||
dateInfo.setUnit(1);
|
||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
semanticParseInfo.setDateInfo(dateInfo);
|
||||
|
||||
// add filter
|
||||
Filter chatFilter = new Filter();
|
||||
chatFilter.setValue(String.valueOf(entity));
|
||||
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
|
||||
chatFilter.setBizName(getEntityPrimaryName(domainInfo));
|
||||
Set<Filter> chatFilters = new LinkedHashSet();
|
||||
chatFilters.add(chatFilter);
|
||||
semanticParseInfo.setDimensionFilters(chatFilters);
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithColumns = null;
|
||||
try {
|
||||
queryResultWithColumns = semanticLayer.queryByStruct(SchemaInfoConverter.convertTo(semanticParseInfo),
|
||||
user);
|
||||
} catch (Exception e) {
|
||||
log.warn("setMainDomain queryByStruct error, e:", e);
|
||||
}
|
||||
|
||||
if (queryResultWithColumns != null) {
|
||||
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
|
||||
&& queryResultWithColumns.getResultList().size() > 0) {
|
||||
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
|
||||
for (Map.Entry<String, Object> entry : result.entrySet()) {
|
||||
String entryKey = getEntryKey(entry);
|
||||
if (entry.getValue() == null || entryKey == null) {
|
||||
continue;
|
||||
}
|
||||
domainInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||
domainInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Set<SchemaItem> getDimensions(EntityInfo domainInfo) {
|
||||
Set<SchemaItem> dimensions = new LinkedHashSet();
|
||||
for (DataInfo mainEntityDimension : domainInfo.getDimensions()) {
|
||||
SchemaItem dimension = new SchemaItem();
|
||||
dimension.setBizName(mainEntityDimension.getBizName());
|
||||
dimensions.add(dimension);
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
private String getEntryKey(Map.Entry<String, Object> entry) {
|
||||
// metric parser special handle, TODO delete
|
||||
String entryKey = entry.getKey();
|
||||
if (entryKey.contains("__")) {
|
||||
entryKey = entryKey.split("__")[1];
|
||||
}
|
||||
return entryKey;
|
||||
}
|
||||
|
||||
private Set<SchemaItem> getMetrics(EntityInfo domainInfo) {
|
||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||
for (DataInfo metricValue : domainInfo.getMetrics()) {
|
||||
SchemaItem metric = new SchemaItem();
|
||||
metric.setBizName(metricValue.getBizName());
|
||||
metrics.add(metric);
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private String getEntityPrimaryName(EntityInfo domainInfo) {
|
||||
return domainInfo.getDomainInfo().getPrimaryEntityBizName();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application;
|
||||
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.*;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import com.tencent.supersonic.chat.application.query.QuerySelector;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.QueryState;
|
||||
import com.tencent.supersonic.chat.domain.service.QueryService;
|
||||
import com.tencent.supersonic.chat.domain.service.ChatService;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@Component("chatQueryService")
|
||||
@Primary
|
||||
@Slf4j
|
||||
public class QueryServiceImpl implements QueryService {
|
||||
|
||||
@Autowired
|
||||
private ChatService chatService;
|
||||
|
||||
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
|
||||
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
private QuerySelector querySelector = ComponentFactory.getQuerySelector();
|
||||
|
||||
@Override
|
||||
public QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception {
|
||||
schemaMappers.stream().forEach(s -> s.map(queryCtx));
|
||||
|
||||
// in order to support multi-turn conversation, we need to consider chat context
|
||||
ChatContext chatCtx = chatService.getOrCreateContext(queryCtx.getChatId());
|
||||
|
||||
for (SemanticParser semanticParser : semanticParsers) {
|
||||
log.info("semanticParser processing:[{}]", semanticParser.getClass().getName());
|
||||
semanticParser.parse(queryCtx, chatCtx);
|
||||
}
|
||||
if (queryCtx.getCandidateQueries().size() > 0) {
|
||||
log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
|
||||
Collectors.toList()));
|
||||
SemanticQuery semanticQuery = querySelector.select(queryCtx.getCandidateQueries());
|
||||
log.info("pick after [{}]", semanticQuery);
|
||||
|
||||
QueryResultResp queryResponse = semanticQuery.execute(queryCtx.getUser());
|
||||
if (queryResponse != null) {
|
||||
// update chat context after a successful semantic query
|
||||
if (queryCtx.isSaveAnswer() && queryResponse.getQueryState() == QueryState.NORMAL.getState()) {
|
||||
chatService.updateContext(chatCtx, queryCtx, semanticQuery.getParseInfo());
|
||||
}
|
||||
queryResponse.setChatContext(chatCtx.getParseInfo());
|
||||
chatService.addQuery(queryResponse, queryCtx, chatCtx);
|
||||
return queryResponse;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticParseInfo queryContext(QueryContextReq queryCtx) {
|
||||
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
|
||||
return context.getParseInfo();
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultResp executeDirectQuery(QueryData queryData, User user) throws Exception {
|
||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||
QueryResultResp queryResponse = new QueryResultResp();
|
||||
BeanUtils.copyProperties(queryData, semanticParseInfo);
|
||||
QueryResultWithSchemaResp resultWithColumns = semanticLayer.queryByStruct(
|
||||
SchemaInfoConverter.convertTo(semanticParseInfo), user);
|
||||
queryResponse.setQueryColumns(resultWithColumns.getColumns());
|
||||
queryResponse.setQueryResults(resultWithColumns.getResultList());
|
||||
return queryResponse;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.RecommendResponse;
|
||||
import com.tencent.supersonic.chat.domain.service.RecommendService;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/***
|
||||
* Recommend Service impl
|
||||
*/
|
||||
@Service
|
||||
public class RecommendServiceImpl implements RecommendService {
|
||||
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
@Override
|
||||
public RecommendResponse recommend(QueryContextReq queryCtx) {
|
||||
Integer domainId = queryCtx.getDomainId();
|
||||
if (Objects.isNull(domainId)) {
|
||||
return new RecommendResponse();
|
||||
}
|
||||
|
||||
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(
|
||||
Long.valueOf(domainId), true);
|
||||
|
||||
List<RecommendResponse.Item> dimensions = domainSchemaDesc.getDimensions().stream().map(dimSchemaDesc -> {
|
||||
RecommendResponse.Item item = new RecommendResponse.Item();
|
||||
item.setDomain(domainId);
|
||||
item.setName(dimSchemaDesc.getName());
|
||||
item.setBizName(dimSchemaDesc.getBizName());
|
||||
return item;
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
List<RecommendResponse.Item> metrics = domainSchemaDesc.getMetrics().stream().map(metricSchemaDesc -> {
|
||||
RecommendResponse.Item item = new RecommendResponse.Item();
|
||||
item.setDomain(domainId);
|
||||
item.setName(metricSchemaDesc.getName());
|
||||
item.setBizName(metricSchemaDesc.getBizName());
|
||||
return item;
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
RecommendResponse response = new RecommendResponse();
|
||||
response.setDimensions(dimensions);
|
||||
response.setMetrics(metrics);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.knowledge;
|
||||
|
||||
import com.tencent.supersonic.common.nlp.WordNature;
|
||||
import com.tencent.supersonic.knowledge.domain.service.OnlineKnowledgeService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.context.event.ApplicationStartedEvent;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class ApplicationStartedInit implements ApplicationListener<ApplicationStartedEvent> {
|
||||
|
||||
@Autowired
|
||||
private OnlineKnowledgeService onlineKnowledgeService;
|
||||
|
||||
@Autowired
|
||||
private WordNatureService wordNatureService;
|
||||
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(ApplicationStartedEvent event) {
|
||||
try {
|
||||
log.info("ApplicationStartedInit start");
|
||||
|
||||
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
|
||||
|
||||
wordNatureService.setPreWordNatures(wordNatures);
|
||||
|
||||
onlineKnowledgeService.reloadAllData(wordNatures);
|
||||
|
||||
log.info("ApplicationStartedInit end");
|
||||
} catch (Exception e) {
|
||||
log.error("ApplicationStartedInit error", e);
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
* reload knowledge task
|
||||
*/
|
||||
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
|
||||
public void reloadKnowledge() {
|
||||
log.info("reloadKnowledge start");
|
||||
|
||||
try {
|
||||
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
|
||||
List<WordNature> preWordNatures = wordNatureService.getPreWordNatures();
|
||||
|
||||
if (CollectionUtils.isEqualCollection(wordNatures, preWordNatures)) {
|
||||
log.debug("wordNatures is not change, reloadKnowledge end");
|
||||
return;
|
||||
}
|
||||
log.info("wordNatures is change");
|
||||
wordNatureService.setPreWordNatures(wordNatures);
|
||||
onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature());
|
||||
wordNatureService.getCache().refresh("");
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("reloadKnowledge error", e);
|
||||
}
|
||||
|
||||
log.info("reloadKnowledge end");
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.knowledge;
|
||||
|
||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.application.mapper.HanlpSchemaMapper;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.DomainInfoStat;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* nature parse helper
|
||||
*/
|
||||
@Slf4j
|
||||
public class NatureHelper {
|
||||
|
||||
private static boolean isDomainOrEntity(Term term, Integer domain) {
|
||||
return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString()
|
||||
.endsWith(NatureType.ENTITY.getType());
|
||||
}
|
||||
|
||||
public static Integer getDomainByNature(Nature nature) {
|
||||
if (nature.startsWith(NatureType.NATURE_SPILT)) {
|
||||
String[] dimensionValues = nature.toString().split(NatureType.NATURE_SPILT);
|
||||
if (StringUtils.isNumeric(dimensionValues[1])) {
|
||||
return Integer.valueOf(dimensionValues[1]);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public static Integer getDomain(String nature) {
|
||||
try {
|
||||
String[] split = nature.split(NatureType.NATURE_SPILT);
|
||||
if (split.length <= 1) {
|
||||
return null;
|
||||
}
|
||||
return Integer.valueOf(split[1]);
|
||||
} catch (NumberFormatException e) {
|
||||
log.error("", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static boolean isDimensionValueClassId(String nature) {
|
||||
if (StringUtils.isEmpty(nature)) {
|
||||
return false;
|
||||
}
|
||||
if (!nature.startsWith(NatureType.NATURE_SPILT)) {
|
||||
return false;
|
||||
}
|
||||
String[] split = nature.split(NatureType.NATURE_SPILT);
|
||||
if (split.length <= 1) {
|
||||
return false;
|
||||
}
|
||||
return !nature.endsWith(NatureType.METRIC.getType()) && !nature.endsWith(NatureType.DIMENSION.getType())
|
||||
&& StringUtils.isNumeric(split[1]);
|
||||
}
|
||||
|
||||
public static DomainInfoStat getDomainStat(List<Term> terms) {
|
||||
DomainInfoStat stat = new DomainInfoStat();
|
||||
stat.setDimensionDomainCount(getDimensionCount(terms));
|
||||
stat.setMetricDomainCount(getMetricCount(terms));
|
||||
stat.setDomainCount(getDomainCount(terms));
|
||||
stat.setDimensionValueDomainCount(getDimensionValueCount(terms));
|
||||
return stat;
|
||||
}
|
||||
|
||||
|
||||
private static long getDomainCount(List<Term> terms) {
|
||||
return terms.stream().filter(term -> isDomainOrEntity(term, getDomainByNature(term.nature))).count();
|
||||
}
|
||||
|
||||
private static long getDimensionValueCount(List<Term> terms) {
|
||||
return terms.stream().filter(term -> isDimensionValueClassId(term.nature.toString())).count();
|
||||
}
|
||||
|
||||
private static long getDimensionCount(List<Term> terms) {
|
||||
return terms.stream().filter(term -> term.nature.startsWith(NatureType.NATURE_SPILT) && term.nature.toString()
|
||||
.endsWith(NatureType.DIMENSION.getType())).count();
|
||||
}
|
||||
|
||||
private static long getMetricCount(List<Term> terms) {
|
||||
return terms.stream().filter(term -> term.nature.startsWith(NatureType.NATURE_SPILT) && term.nature.toString()
|
||||
.endsWith(NatureType.METRIC.getType())).count();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of types of class parts of speech
|
||||
* domainId -> (nature , natureCount)
|
||||
*
|
||||
* @param terms
|
||||
* @return
|
||||
*/
|
||||
public static Map<Integer, Map<NatureType, Integer>> getDomainToNatureStat(List<Term> terms) {
|
||||
Map<Integer, Map<NatureType, Integer>> domainToNature = new HashMap<>();
|
||||
terms.stream().filter(
|
||||
term -> term.nature.startsWith(NatureType.NATURE_SPILT)
|
||||
).forEach(term -> {
|
||||
NatureType natureType = NatureType.getNatureType(String.valueOf(term.nature));
|
||||
Integer domain = getDomain(String.valueOf(term.nature));
|
||||
|
||||
Map<NatureType, Integer> natureTypeMap = new HashMap<>();
|
||||
natureTypeMap.put(natureType, 1);
|
||||
|
||||
Map<NatureType, Integer> original = domainToNature.get(domain);
|
||||
if (Objects.isNull(original)) {
|
||||
domainToNature.put(domain, natureTypeMap);
|
||||
} else {
|
||||
Integer count = original.get(natureType);
|
||||
if (Objects.isNull(count)) {
|
||||
count = 1;
|
||||
} else {
|
||||
count = count + 1;
|
||||
}
|
||||
original.put(natureType, count);
|
||||
}
|
||||
});
|
||||
return domainToNature;
|
||||
}
|
||||
|
||||
public static List<Integer> selectPossibleDomains(List<Term> terms) {
|
||||
Map<Integer, Map<NatureType, Integer>> domainToNatureStat = getDomainToNatureStat(terms);
|
||||
Integer maxDomainTypeSize = domainToNatureStat.entrySet().stream()
|
||||
.max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size())
|
||||
.orElse(null);
|
||||
if (Objects.isNull(maxDomainTypeSize) || maxDomainTypeSize == 0) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return domainToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxDomainTypeSize)
|
||||
.map(entry -> entry.getKey()).collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.knowledge;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
import com.tencent.supersonic.common.nlp.WordNature;
|
||||
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
/**
|
||||
* word nature service
|
||||
**/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class WordNatureService {
|
||||
|
||||
private static final Integer META_CACHE_TIME = 5;
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
private List<WordNature> preWordNatures = new ArrayList<>();
|
||||
|
||||
private LoadingCache<String, DomainInfos> cache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
|
||||
.build(
|
||||
new CacheLoader<String, DomainInfos>() {
|
||||
@Override
|
||||
public DomainInfos load(String key) {
|
||||
log.info("load getDomainSchemaInfo cache [{}]", key);
|
||||
return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
public List<WordNature> getAllWordNature() {
|
||||
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
DomainInfos domainInfos = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||
|
||||
List<WordNature> natures = new ArrayList<>();
|
||||
|
||||
addNatureToResult(NatureType.DIMENSION, domainInfos.getDimensions(), natures);
|
||||
|
||||
addNatureToResult(NatureType.METRIC, domainInfos.getMetrics(), natures);
|
||||
|
||||
addNatureToResult(NatureType.DOMAIN, domainInfos.getDomains(), natures);
|
||||
|
||||
addNatureToResult(NatureType.ENTITY, domainInfos.getEntities(), natures);
|
||||
|
||||
return natures;
|
||||
}
|
||||
|
||||
private void addNatureToResult(NatureType value, List<ItemDO> metas, List<WordNature> natures) {
|
||||
List<WordNature> natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas);
|
||||
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
|
||||
natures.addAll(natureList);
|
||||
}
|
||||
|
||||
public List<WordNature> getPreWordNatures() {
|
||||
return preWordNatures;
|
||||
}
|
||||
|
||||
public void setPreWordNatures(List<WordNature> preWordNatures) {
|
||||
this.preWordNatures = preWordNatures;
|
||||
}
|
||||
|
||||
public LoadingCache<String, DomainInfos> getCache() {
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class DatabaseSchemaMapper implements SchemaMapper {
|
||||
|
||||
@Override
|
||||
public void map(QueryContextReq queryContext) {
|
||||
|
||||
log.debug("before db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||
|
||||
List<Term> terms = HanlpHelper.getTerms(queryContext.getQueryText());
|
||||
|
||||
WordNatureService wordNatureService = ContextUtils.getBean(WordNatureService.class);
|
||||
|
||||
DomainInfos domainInfos = wordNatureService.getCache().getUnchecked("");
|
||||
|
||||
detectAndAddToSchema(queryContext, terms, domainInfos.getDimensions(),
|
||||
SchemaElementType.DIMENSION);
|
||||
detectAndAddToSchema(queryContext, terms, domainInfos.getMetrics(), SchemaElementType.METRIC);
|
||||
|
||||
log.debug("after db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||
}
|
||||
|
||||
private void detectAndAddToSchema(QueryContextReq queryContext, List<Term> terms, List<ItemDO> domains,
|
||||
SchemaElementType schemaElementType) {
|
||||
try {
|
||||
String queryText = queryContext.getQueryText();
|
||||
|
||||
Map<String, Set<ItemDO>> domainResultSet = getResultSet(queryText, terms, domains);
|
||||
|
||||
addToSchemaMapInfo(domainResultSet, queryContext.getMapInfo(), schemaElementType);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("detectAndAddToSchema error", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Set<ItemDO>> getResultSet(String queryText, List<Term> terms, List<ItemDO> domains) {
|
||||
|
||||
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||
|
||||
Map<String, Set<ItemDO>> nameToItems = getNameToItems(domains);
|
||||
|
||||
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
|
||||
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
|
||||
|
||||
Map<String, Set<ItemDO>> domainResultSet = new HashMap<>();
|
||||
for (Integer index = 0; index <= queryText.length() - 1; ) {
|
||||
for (Integer i = index; i <= queryText.length(); ) {
|
||||
i = mapperHelper.getStepIndex(regOffsetToLength, i);
|
||||
if (i <= queryText.length()) {
|
||||
String detectSegment = queryText.substring(index, i);
|
||||
nameToItems.forEach(
|
||||
(name, newItemDOs) -> {
|
||||
if (name.contains(detectSegment)
|
||||
&& mapperHelper.getSimilarity(detectSegment, name)
|
||||
>= mapperHelper.getMetricDimensionThresholdConfig()) {
|
||||
Set<ItemDO> preItemDOS = domainResultSet.putIfAbsent(detectSegment, newItemDOs);
|
||||
if (Objects.nonNull(preItemDOS)) {
|
||||
preItemDOS.addAll(newItemDOs);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
index = mapperHelper.getStepIndex(regOffsetToLength, index);
|
||||
}
|
||||
return domainResultSet;
|
||||
}
|
||||
|
||||
private Map<String, Set<ItemDO>> getNameToItems(List<ItemDO> domains) {
|
||||
return domains.stream()
|
||||
.collect(Collectors.toMap(ItemDO::getName, a -> {
|
||||
Set<ItemDO> result = new HashSet<>();
|
||||
result.add(a);
|
||||
return result;
|
||||
}, (k1, k2) -> {
|
||||
k1.addAll(k2);
|
||||
return k1;
|
||||
}));
|
||||
}
|
||||
|
||||
private void addToSchemaMapInfo(Map<String, Set<ItemDO>> mapResultRowSet, SchemaMapInfo schemaMap,
|
||||
SchemaElementType schemaElementType) {
|
||||
if (Objects.isNull(mapResultRowSet) || mapResultRowSet.size() <= 0) {
|
||||
return;
|
||||
}
|
||||
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||
|
||||
for (Map.Entry<String, Set<ItemDO>> entry : mapResultRowSet.entrySet()) {
|
||||
String detectWord = entry.getKey();
|
||||
Set<ItemDO> itemDOS = entry.getValue();
|
||||
for (ItemDO itemDO : itemDOS) {
|
||||
|
||||
List<SchemaElementMatch> elements = schemaMap.getMatchedElements(itemDO.getDomain());
|
||||
if (CollectionUtils.isEmpty(elements)) {
|
||||
elements = new ArrayList<>();
|
||||
schemaMap.setMatchedElements(itemDO.getDomain(), elements);
|
||||
}
|
||||
Set<Integer> regElementSet = elements.stream()
|
||||
.filter(elementMatch -> schemaElementType.equals(elementMatch.getElementType()))
|
||||
.map(elementMatch -> elementMatch.getElementID())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
if (regElementSet.contains(itemDO.getItemId())) {
|
||||
continue;
|
||||
}
|
||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||
.elementID(itemDO.getItemId()).word(itemDO.getName()).frequency(10000L)
|
||||
.elementType(schemaElementType).detectWord(detectWord)
|
||||
.similarity(mapperHelper.getSimilarity(detectWord, itemDO.getName()))
|
||||
.build();
|
||||
log.info("schemaElementType:{},add to schema, elementMatch {}", schemaElementType, schemaElementMatch);
|
||||
elements.add(schemaElementMatch);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||
import com.tencent.supersonic.chat.domain.utils.NatureConverter;
|
||||
import com.tencent.supersonic.common.nlp.MapResult;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
|
||||
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
|
||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class HanlpSchemaMapper implements SchemaMapper {
|
||||
|
||||
@Override
|
||||
public void map(QueryContextReq queryContext) {
|
||||
|
||||
List<Term> terms = HanlpHelper.getTerms(queryContext.getQueryText());
|
||||
|
||||
terms.forEach(
|
||||
item -> log.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
|
||||
item.getFrequency())
|
||||
);
|
||||
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
|
||||
|
||||
Map<MatchText, List<MapResult>> matchResult = matchStrategy.match(queryContext.getQueryText(), terms,
|
||||
queryContext.getDomainId());
|
||||
List<MapResult> matches = new ArrayList<>();
|
||||
if (Objects.nonNull(matchResult)) {
|
||||
Optional<List<MapResult>> first = matchResult.entrySet().stream()
|
||||
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
||||
.map(entry -> entry.getValue()).findFirst();
|
||||
if (first.isPresent()) {
|
||||
matches = first.get();
|
||||
}
|
||||
}
|
||||
HanlpHelper.transLetterOriginal(matches);
|
||||
log.info("queryContext:{},matches:{}", queryContext, matches);
|
||||
|
||||
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap, List<Term> terms) {
|
||||
if (CollectionUtils.isEmpty(mapResults)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Long> wordNatureToFrequency = terms.stream().collect(
|
||||
Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
|
||||
term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
|
||||
|
||||
for (MapResult mapResult : mapResults) {
|
||||
for (String nature : mapResult.getNatures()) {
|
||||
Integer domain = NatureHelper.getDomain(nature);
|
||||
if (Objects.isNull(domain)) {
|
||||
continue;
|
||||
}
|
||||
SchemaElementType elementType = NatureConverter.convertTo(nature);
|
||||
if (Objects.isNull(elementType)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature));
|
||||
Integer elementID = baseWordNature.getElementID(nature);
|
||||
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
|
||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||
.elementType(elementType)
|
||||
.elementID(elementID)
|
||||
.frequency(frequency)
|
||||
.word(mapResult.getName())
|
||||
.similarity(mapResult.getSimilarity())
|
||||
.detectWord(mapResult.getDetectWord())
|
||||
.build();
|
||||
|
||||
Map<Integer, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
|
||||
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domain,
|
||||
new ArrayList<>());
|
||||
if (schemaElementMatches == null) {
|
||||
schemaElementMatches = domainElementMatches.get(domain);
|
||||
}
|
||||
schemaElementMatches.add(schemaElementMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Slf4j
|
||||
public class QueryFilterMapper implements SchemaMapper {
|
||||
|
||||
private Long FREQUENCY = 9999999L;
|
||||
|
||||
private double SIMILARITY = 1.0;
|
||||
|
||||
@Override
|
||||
public void map(QueryContextReq queryContext) {
|
||||
Integer domainId = queryContext.getDomainId();
|
||||
if (domainId == null || domainId <= 0 || queryContext.getQueryFilter() == null) {
|
||||
return;
|
||||
}
|
||||
QueryFilter queryFilter = queryContext.getQueryFilter();
|
||||
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
|
||||
clearOtherSchemaElementMatch(domainId, schemaMapInfo);
|
||||
convertFilterToSchemaMapInfo(queryFilter.getFilters(), schemaElementMatches);
|
||||
}
|
||||
|
||||
private void convertFilterToSchemaMapInfo(List<Filter> filters, List<SchemaElementMatch> schemaElementMatches) {
|
||||
log.info("schemaElementMatches before queryFilerMapper:{}", schemaElementMatches);
|
||||
if (CollectionUtils.isEmpty(schemaElementMatches)) {
|
||||
schemaElementMatches = Lists.newArrayList();
|
||||
}
|
||||
List<String> words = schemaElementMatches.stream().map(SchemaElementMatch::getWord).collect(Collectors.toList());
|
||||
for (Filter filter : filters) {
|
||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||
.elementType(SchemaElementType.VALUE)
|
||||
.elementID(filter.getElementID().intValue())
|
||||
.frequency(FREQUENCY)
|
||||
.word(String.valueOf(filter.getValue()))
|
||||
.similarity(SIMILARITY)
|
||||
.detectWord(Constants.EMPTY)
|
||||
.build();
|
||||
if (words.contains(schemaElementMatch.getWord())) {
|
||||
continue;
|
||||
}
|
||||
schemaElementMatches.add(schemaElementMatch);
|
||||
}
|
||||
log.info("schemaElementMatches after queryFilerMapper:{}", schemaElementMatches);
|
||||
}
|
||||
|
||||
private void clearOtherSchemaElementMatch(Integer domainId, SchemaMapInfo schemaMapInfo) {
|
||||
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
|
||||
if (!entry.getKey().equals(domainId)) {
|
||||
entry.getValue().clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.query.EntityListFilter;
|
||||
import com.tencent.supersonic.chat.application.query.MetricGroupBy;
|
||||
import com.tencent.supersonic.chat.application.query.MetricOrderBy;
|
||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class AggregateSemanticParser implements SemanticParser {
|
||||
|
||||
public static final Integer TOPN_LIMIT = 1000;
|
||||
|
||||
private static Map<AggregateTypeEnum, Pattern> aggregateRegexMap = new HashMap<>();
|
||||
|
||||
static {
|
||||
aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高|最多)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低|最少)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)"));
|
||||
aggregateRegexMap.put(AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)"));
|
||||
}
|
||||
|
||||
public static AggregateTypeEnum resolveAggregateType(String queryText) {
|
||||
|
||||
Map<AggregateTypeEnum, Integer> aggregateCount = new HashMap<>(aggregateRegexMap.size());
|
||||
for (Map.Entry<AggregateTypeEnum, Pattern> entry : aggregateRegexMap.entrySet()) {
|
||||
Matcher matcher = entry.getValue().matcher(queryText);
|
||||
int count = 0;
|
||||
while (matcher.find()) {
|
||||
count++;
|
||||
}
|
||||
if (count > 0) {
|
||||
aggregateCount.put(entry.getKey(), count);
|
||||
}
|
||||
}
|
||||
|
||||
return aggregateCount.entrySet().stream().max(Map.Entry.comparingByValue()).map(entry -> entry.getKey())
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||
AggregateTypeEnum aggregateType = resolveAggregateType(queryContext.getQueryText());
|
||||
|
||||
for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) {
|
||||
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
|
||||
|
||||
semanticParse.setNativeQuery(getNativeQuery(aggregateType, semanticParse));
|
||||
semanticParse.setAggType(aggregateType);
|
||||
if (Objects.isNull(semanticParse.getLimit()) || semanticParse.getLimit() <= 0) {
|
||||
semanticParse.setLimit(Long.valueOf(TOPN_LIMIT));
|
||||
}
|
||||
resetQueryModeByAggregateType(semanticParse, aggregateType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* query mode reset by the AggregateType
|
||||
*
|
||||
* @param parseInfo
|
||||
* @param aggregateType
|
||||
*/
|
||||
private void resetQueryModeByAggregateType(SemanticParseInfo parseInfo,
|
||||
AggregateTypeEnum aggregateType) {
|
||||
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) {
|
||||
if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType)
|
||||
|| AggregateTypeEnum.TOPN.equals(aggregateType)) {
|
||||
parseInfo.setQueryMode(MetricOrderBy.QUERY_MODE);
|
||||
} else {
|
||||
parseInfo.setQueryMode(MetricGroupBy.QUERY_MODE);
|
||||
}
|
||||
log.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean getNativeQuery(AggregateTypeEnum aggregateType, SemanticParseInfo semanticParse) {
|
||||
if (AggregateTypeEnum.TOPN.equals(aggregateType)) {
|
||||
return true;
|
||||
}
|
||||
if (EntityListFilter.QUERY_MODE.equals(semanticParse.getQueryMode()) && (semanticParse.getMetrics() == null
|
||||
|| semanticParse.getMetrics().isEmpty())) {
|
||||
return true;
|
||||
}
|
||||
return semanticParse.getNativeQuery();
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public interface DomainResolver {
|
||||
|
||||
Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq queryCtx, ChatContext chatCtx,
|
||||
SchemaMapInfo schemaMap);
|
||||
|
||||
boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo);
|
||||
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.query.EntitySemanticQuery;
|
||||
import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
|
||||
import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
|
||||
import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.domain.service.ConfigService;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import com.tencent.supersonic.chat.domain.utils.DefaultMetricUtils;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class DomainSemanticParser implements SemanticParser {
|
||||
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
@Override
|
||||
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||
DomainInfos domainInfosDb = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||
Map<Integer, String> domainToName = domainInfosDb.getDomainToName();
|
||||
SchemaMapInfo mapInfo = queryContext.getMapInfo();
|
||||
|
||||
// iterate all schemaElementMatches to resolve semantic query
|
||||
for (Integer domainId : mapInfo.getMatchedDomains()) {
|
||||
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(domainId);
|
||||
List<RuleSemanticQuery> queries = resolveQuery(elementMatches, queryContext);
|
||||
for (RuleSemanticQuery query : queries) {
|
||||
|
||||
if (useBlackItem(query, domainId)) {
|
||||
log.info("useBlackItem, skip query:{}", query);
|
||||
continue;
|
||||
}
|
||||
addCandidateQuery(queryContext, chatContext, domainId.longValue(),
|
||||
domainToName.get(domainId), query);
|
||||
}
|
||||
}
|
||||
|
||||
// if no candidates have been found yet, count in chat context and try again
|
||||
if (queryContext.getCandidateQueries().size() <= 0) {
|
||||
if (chatContext.getParseInfo() != null && chatContext.getParseInfo().getDomainId() > 0) {
|
||||
Integer chatDomainId = Integer.valueOf(chatContext.getParseInfo().getDomainId().intValue());
|
||||
if (mapInfo.getMatchedDomains().contains(chatDomainId)) {
|
||||
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(chatDomainId);
|
||||
|
||||
List<RuleSemanticQuery> queries = tryParseByContext(elementMatches, chatContext, queryContext);
|
||||
for (RuleSemanticQuery query : queries) {
|
||||
addCandidateQuery(queryContext, chatContext, chatDomainId.longValue(),
|
||||
domainToName.get(chatDomainId), query);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean useBlackItem(RuleSemanticQuery query, Integer domainId) {
|
||||
if (Objects.isNull(domainId)) {
|
||||
return false;
|
||||
}
|
||||
ConfigService configService = ContextUtils.getBean(ConfigService.class);
|
||||
ChatConfigResp chatConfigResp = configService.fetchConfigByDomainId(domainId.longValue());
|
||||
if (Objects.nonNull(chatConfigResp) && Objects.nonNull(query) && Objects.nonNull(query.getParseInfo())) {
|
||||
List<SchemaElementMatch> elementMatches = query.getParseInfo().getElementMatches();
|
||||
if (!CollectionUtils.isEmpty(elementMatches)) {
|
||||
return useBlackItemInternal(elementMatches, chatConfigResp, query);
|
||||
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean useBlackItemInternal(List<SchemaElementMatch> elementMatches, ChatConfigResp chatConfigResp, RuleSemanticQuery query) {
|
||||
if (Objects.isNull(chatConfigResp)) {
|
||||
return false;
|
||||
}
|
||||
List<Long> blackDimIdList = new ArrayList<>();
|
||||
List<Long> blackMetricIdList = new ArrayList<>();
|
||||
if (query instanceof EntitySemanticQuery
|
||||
&& Objects.nonNull(chatConfigResp.getChatDetailConfig())
|
||||
&& Objects.nonNull(chatConfigResp.getChatDetailConfig().getVisibility())) {
|
||||
log.info("useBlackItem, handle EntitySemanticQuery blackList logic");
|
||||
blackDimIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackDimIdList();
|
||||
blackMetricIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackMetricIdList();
|
||||
}
|
||||
|
||||
if (query instanceof MetricSemanticQuery
|
||||
&& Objects.nonNull(chatConfigResp.getChatAggConfig())
|
||||
&& Objects.nonNull(chatConfigResp.getChatAggConfig().getVisibility())) {
|
||||
log.info("useBlackItem, handle MetricSemanticQuery blackList logic");
|
||||
blackDimIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackDimIdList();
|
||||
blackMetricIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackMetricIdList();
|
||||
}
|
||||
return useBlackItemWithElementMatches(elementMatches, blackDimIdList, blackMetricIdList);
|
||||
}
|
||||
|
||||
private boolean useBlackItemWithElementMatches(List<SchemaElementMatch> elementMatches, List<Long> blackDimIdList, List<Long> blackMetricIdList) {
|
||||
|
||||
Set<Long> dimIds = elementMatches.stream()
|
||||
.filter(element -> SchemaElementType.VALUE.equals(element.getElementType()) || SchemaElementType.DIMENSION.equals(element.getElementType()))
|
||||
.map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
|
||||
|
||||
Set<Long> metricIds = elementMatches.stream()
|
||||
.filter(element -> SchemaElementType.METRIC.equals(element.getElementType()))
|
||||
.map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
|
||||
|
||||
|
||||
return useBlackItemWithIds(dimIds, metricIds, blackDimIdList, blackMetricIdList);
|
||||
}
|
||||
|
||||
private boolean useBlackItemWithIds(Set<Long> dimIds, Set<Long> metricIds, List<Long> blackDimIdList, List<Long> blackMetricIdList) {
|
||||
|
||||
if (!CollectionUtils.isEmpty(blackDimIdList) && !CollectionUtils.isEmpty(dimIds)) {
|
||||
if (blackDimIdList.stream().anyMatch(dimIds::contains)) {
|
||||
log.info("useBlackItem, blackDimIdList:{}", blackDimIdList.stream().filter(dimIds::contains).collect(Collectors.toList()));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(blackMetricIdList) && !CollectionUtils.isEmpty(metricIds)) {
|
||||
if (blackMetricIdList.stream().anyMatch(metricIds::contains)) {
|
||||
log.info("useBlackItem, blackMetricIdList:{}", blackMetricIdList.stream().filter(metricIds::contains).collect(Collectors.toList()));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void addCandidateQuery(QueryContextReq queryContext, ChatContext chatContext,
|
||||
Long domainId, String domainName, RuleSemanticQuery semanticQuery) {
|
||||
if (semanticQuery != null) {
|
||||
DefaultMetricUtils defaultMetricUtils = ContextUtils.getBean(DefaultMetricUtils.class);
|
||||
defaultMetricUtils.fillParseInfo(semanticQuery, domainId, domainName);
|
||||
inheritContext(semanticQuery, chatContext);
|
||||
defaultMetricUtils.fillDefaultMetric(semanticQuery.getParseInfo(), queryContext, chatContext);
|
||||
queryContext.getCandidateQueries().add(semanticQuery);
|
||||
}
|
||||
}
|
||||
|
||||
protected void inheritContext(RuleSemanticQuery semanticQuery, ChatContext chatContext) {
|
||||
// is domain switch
|
||||
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
|
||||
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
|
||||
if (!domainResolver.isDomainSwitch(chatContext, semanticParse)) {
|
||||
semanticQuery.inheritContext(chatContext);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* try to add ChatContext to SchemaMatch and look if match QueryMode
|
||||
*
|
||||
* @param elementMatches
|
||||
* @param chatCtx
|
||||
* @return
|
||||
*/
|
||||
private List<RuleSemanticQuery> tryParseByContext(List<SchemaElementMatch> elementMatches,
|
||||
ChatContext chatCtx, QueryContextReq queryCtx) {
|
||||
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getEntity() > 0) {
|
||||
Long entityCount = elementMatches.stream().filter(i -> SchemaElementType.ENTITY.equals(i.getElementType()))
|
||||
.count();
|
||||
Long metricCount = elementMatches.stream().filter(i -> SchemaElementType.METRIC.equals(i.getElementType()))
|
||||
.count();
|
||||
if (entityCount <= 0 && metricCount <= 0 && ContextHelper.hasEntityId(chatCtx)) {
|
||||
// try entity parse
|
||||
SchemaElementMatch entityElementMatch = SchemaElementMatch.builder()
|
||||
.elementType(SchemaElementType.ENTITY).build();
|
||||
List<SchemaElementMatch> newSchemaMatches = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(elementMatches)) {
|
||||
newSchemaMatches.addAll(elementMatches);
|
||||
}
|
||||
newSchemaMatches.add(entityElementMatch);
|
||||
List<RuleSemanticQuery> queries = doParseByContext(newSchemaMatches, chatCtx, queryCtx);
|
||||
if (queries.size() > 0) {
|
||||
return queries;
|
||||
}
|
||||
}
|
||||
}
|
||||
return doParseByContext(elementMatches, chatCtx, queryCtx);
|
||||
}
|
||||
|
||||
|
||||
private List<RuleSemanticQuery> doParseByContext(List<SchemaElementMatch> elementMatches,
|
||||
ChatContext chatCtx, QueryContextReq queryContext) {
|
||||
SemanticParseInfo contextSemanticParse = chatCtx.getParseInfo();
|
||||
if (contextSemanticParse != null) {
|
||||
List<SchemaElementMatch> newElementMatches = new ArrayList<>();
|
||||
List<List<SchemaElementType>> trySchemaElementTypes = new LinkedList<>();
|
||||
// try DIMENSION+METRIC+VALUE
|
||||
// try DIMENSION+METRIC METRIC+VALUE DIMENSION+VALUE
|
||||
// try DIMENSION METRIC VALUE single
|
||||
trySchemaElementTypes.add(new ArrayList<>(
|
||||
Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.METRIC, SchemaElementType.VALUE)));
|
||||
trySchemaElementTypes.add(
|
||||
new ArrayList<>(Arrays.asList(SchemaElementType.METRIC, SchemaElementType.VALUE)));
|
||||
trySchemaElementTypes.add(
|
||||
new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.METRIC)));
|
||||
trySchemaElementTypes.add(
|
||||
new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.VALUE)));
|
||||
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.METRIC)));
|
||||
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.VALUE)));
|
||||
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION)));
|
||||
|
||||
for (List<SchemaElementType> schemaTypes : trySchemaElementTypes) {
|
||||
newElementMatches.clear();
|
||||
if (!CollectionUtils.isEmpty(elementMatches)) {
|
||||
newElementMatches.addAll(elementMatches);
|
||||
}
|
||||
ContextHelper.mergeContextSchemaElementMatch(newElementMatches, elementMatches, schemaTypes,
|
||||
contextSemanticParse);
|
||||
List<RuleSemanticQuery> queries = resolveQuery(newElementMatches, queryContext);
|
||||
if (queries.size() > 0) {
|
||||
return queries;
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private List<RuleSemanticQuery> resolveQuery(List<SchemaElementMatch> candidateElementMatches,
|
||||
QueryContextReq queryContext) {
|
||||
List<RuleSemanticQuery> matchedQueries = new ArrayList<>();
|
||||
for (RuleSemanticQuery semanticQuery : RuleSemanticQueryManager.getSemanticQueries()) {
|
||||
List<SchemaElementMatch> matches = semanticQuery.match(candidateElementMatches, queryContext);
|
||||
|
||||
if (matches.size() > 0) {
|
||||
log.info("resolve match [{}:{}] ", semanticQuery.getQueryMode(), matches.size());
|
||||
RuleSemanticQuery query = RuleSemanticQueryManager.create(semanticQuery.getQueryMode());
|
||||
query.getParseInfo().getElementMatches().addAll(matches);
|
||||
matchedQueries.add(query);
|
||||
}
|
||||
}
|
||||
|
||||
return matchedQueries;
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
public class HeuristicDomainResolver implements DomainResolver {
|
||||
|
||||
protected static Integer selectDomainBySchemaElementCount(Map<Integer, SemanticQuery> domainQueryModes,
|
||||
SchemaMapInfo schemaMap) {
|
||||
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
|
||||
if (domainTypeMap.size() == 1) {
|
||||
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
|
||||
if (domainQueryModes.containsKey(domainSelect)) {
|
||||
log.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
|
||||
return domainSelect;
|
||||
}
|
||||
} else {
|
||||
Map.Entry<Integer, QueryMatchInfo> maxDomain = domainTypeMap.entrySet().stream()
|
||||
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
|
||||
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
|
||||
if (maxDomain != null) {
|
||||
log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey());
|
||||
return maxDomain.getKey();
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* to check can switch domain if context exit domain
|
||||
*
|
||||
* @return false will use context domain, true will use other domain , maybe include context domain
|
||||
*/
|
||||
protected static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
|
||||
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
|
||||
if (!Objects.nonNull(domainId) || domainId <= 0) {
|
||||
return true;
|
||||
}
|
||||
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
|
||||
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
|
||||
log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
|
||||
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
|
||||
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
|
||||
.filter(entry -> entry.getValue().getCount() > 1).count();
|
||||
if (otherDomainTypeNumBigOneCount >= 1) {
|
||||
return true;
|
||||
}
|
||||
// if query text only contain time , will not switch
|
||||
for (SemanticQuery semanticQuery : domainQueryModes.values()) {
|
||||
SemanticParseInfo semanticParseInfo = semanticQuery.getParseInfo();
|
||||
if (semanticParseInfo == null) {
|
||||
continue;
|
||||
}
|
||||
if (searchCtx.getQueryText() != null && semanticParseInfo.getDateInfo() != null) {
|
||||
if (semanticParseInfo.getDateInfo().getText() != null) {
|
||||
if (semanticParseInfo.getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
|
||||
log.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
|
||||
semanticParseInfo.getDateInfo());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if context domain not in schemaMap , will switch
|
||||
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
|
||||
log.info("domainId not in schemaMap ");
|
||||
return true;
|
||||
}
|
||||
// other will not switch
|
||||
return false;
|
||||
}
|
||||
|
||||
public static Map<Integer, QueryMatchInfo> getDomainTypeMap(SchemaMapInfo schemaMap) {
|
||||
Map<Integer, QueryMatchInfo> domainCount = new HashMap<>();
|
||||
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
|
||||
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
|
||||
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
|
||||
if (!domainCount.containsKey(entry.getKey())) {
|
||||
domainCount.put(entry.getKey(), new QueryMatchInfo());
|
||||
}
|
||||
QueryMatchInfo queryMatchInfo = domainCount.get(entry.getKey());
|
||||
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
|
||||
schemaElementMatches.stream()
|
||||
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
|
||||
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
|
||||
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
|
||||
if (schemaElementMatchMax != null) {
|
||||
queryMatchInfo.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
|
||||
}
|
||||
queryMatchInfo.setCount(schemaElementTypes.size());
|
||||
|
||||
}
|
||||
}
|
||||
return domainCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo) {
|
||||
Long contextDomain = chatCtx.getParseInfo().getDomainId();
|
||||
Long currentDomain = semanticParseInfo.getDomainId();
|
||||
boolean noSwitch =
|
||||
currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
|
||||
log.debug("ChatContext isDomainSwitch [{}] [{}]",
|
||||
semanticParseInfo.getQueryMode(), !noSwitch);
|
||||
return !noSwitch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
||||
ChatContext chatCtx, SchemaMapInfo schemaMap) {
|
||||
Integer selectDomain = selectDomain(domainQueryModes, searchCtx, chatCtx, schemaMap);
|
||||
if (selectDomain > 0) {
|
||||
log.info("selectDomain {} ", selectDomain);
|
||||
return selectDomain;
|
||||
}
|
||||
// get the max SchemaElementType number
|
||||
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
|
||||
}
|
||||
|
||||
public Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
||||
ChatContext chatCtx,
|
||||
SchemaMapInfo schemaMap) {
|
||||
// if QueryContext has domainId and in domainQueryModes
|
||||
if (domainQueryModes.containsKey(searchCtx.getDomainId())) {
|
||||
log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
|
||||
return searchCtx.getDomainId();
|
||||
}
|
||||
// if ChatContext has domainId and in domainQueryModes
|
||||
if (chatCtx.getParseInfo().getDomainId() > 0) {
|
||||
Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
|
||||
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) {
|
||||
log.info("selectDomain from ChatContext [{}]", domainId);
|
||||
return domainId;
|
||||
}
|
||||
}
|
||||
// default 0
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
|
||||
import com.tencent.supersonic.chat.application.query.LLMSemanticQuery;
|
||||
import com.tencent.supersonic.chat.domain.config.LLMConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.LLMReq;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.LLMResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.LLMSchema;
|
||||
import com.tencent.supersonic.chat.domain.utils.DslToSemanticInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.SemanticSatisfactionChecker;
|
||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.calcite.sql.parser.SqlParseException;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
@Slf4j
|
||||
public class LLMSemanticParser implements SemanticParser {
|
||||
|
||||
private DslToSemanticInfo dslToSemanticInfo = new DslToSemanticInfo();
|
||||
|
||||
@Override
|
||||
public void parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
||||
String queryText = queryContext.getQueryText();
|
||||
|
||||
if (SemanticSatisfactionChecker.check(queryContext)) {
|
||||
log.info("There is no need parse by llm , queryText:{}", queryText);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Integer domainId = getDomainId(queryContext, chatCtx);
|
||||
LLMResp llmResp = requestLLM(queryContext, domainId);
|
||||
if (Objects.isNull(llmResp)) {
|
||||
return;
|
||||
}
|
||||
LLMSemanticQuery semanticQuery = new LLMSemanticQuery();
|
||||
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||
String sql = convertToSql(llmResp, parseInfo, domainId);
|
||||
|
||||
parseInfo.setInfo(sql);
|
||||
parseInfo.setDomainId(Long.valueOf(domainId));
|
||||
parseInfo.setBonus(queryText.length() * 1.0);
|
||||
parseInfo.setQueryMode(LLMSemanticQuery.QUERY_MODE);
|
||||
queryContext.getCandidateQueries().add(semanticQuery);
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
log.error("llm parse error , skip the parser. error:", e);
|
||||
}
|
||||
}
|
||||
|
||||
protected String convertToSql(LLMResp llmResp, SemanticParseInfo parseInfo, Integer domainId)
|
||||
throws SqlParseException {
|
||||
return dslToSemanticInfo.convert(parseInfo, llmResp, domainId);
|
||||
}
|
||||
|
||||
protected LLMResp requestLLM(QueryContextReq queryContext, Integer domainId) {
|
||||
final LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class);
|
||||
|
||||
if (StringUtils.isEmpty(llmConfig.getUrl())) {
|
||||
log.warn("llmConfig url is null, skip llm parser");
|
||||
return null;
|
||||
}
|
||||
|
||||
DomainInfos domainInfos = ContextUtils.getBean(WordNatureService.class).getCache().getUnchecked("");
|
||||
|
||||
Map<Integer, String> domainIdToName = domainInfos.getDomains().stream()
|
||||
.collect(Collectors.toMap(ItemDO::getDomain, a -> a.getName(), (k1, k2) -> k1));
|
||||
|
||||
Map<Integer, String> itemIdToName = domainInfos.getDimensions().stream()
|
||||
.filter(entry -> domainId.equals(entry.getDomain()))
|
||||
.collect(Collectors.toMap(ItemDO::getItemId, ItemDO::getName, (value1, value2) -> value2));
|
||||
|
||||
String domainName = domainIdToName.get(domainId);
|
||||
LLMReq llmReq = new LLMReq();
|
||||
llmReq.setQueryText(queryContext.getQueryText());
|
||||
|
||||
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
|
||||
|
||||
Set<String> fieldNameList = matchedElements.stream()
|
||||
.filter(schemaElementMatch ->
|
||||
SchemaElementType.METRIC.equals(schemaElementMatch.getElementType()) ||
|
||||
SchemaElementType.DIMENSION.equals(schemaElementMatch.getElementType()) ||
|
||||
SchemaElementType.VALUE.equals(schemaElementMatch.getElementType()))
|
||||
.map(schemaElementMatch -> {
|
||||
if (!SchemaElementType.VALUE.equals(schemaElementMatch.getElementType())) {
|
||||
return schemaElementMatch.getWord();
|
||||
}
|
||||
return itemIdToName.get(schemaElementMatch.getElementID());
|
||||
})
|
||||
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
LLMSchema llmSchema = new LLMSchema();
|
||||
llmSchema.setDomainName(domainName);
|
||||
llmSchema.setFieldNameList(new ArrayList<>(fieldNameList));
|
||||
llmReq.setSchema(llmSchema);
|
||||
|
||||
log.info("requestLLM request, domainId:{},llmReq:{}", domainId, llmReq);
|
||||
String questUrl = llmConfig.getUrl() + llmConfig.getQueryToSqlPath();
|
||||
|
||||
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
HttpEntity<String> entity = new HttpEntity<>(JsonUtil.toString(llmReq), headers);
|
||||
|
||||
ResponseEntity<LLMResp> responseEntity = restTemplate.exchange(questUrl, HttpMethod.POST, entity,
|
||||
LLMResp.class);
|
||||
|
||||
log.info("requestLLM response, questUrl:{} \n entity:{} \n body:{}", questUrl, entity,
|
||||
responseEntity.getBody());
|
||||
return responseEntity.getBody();
|
||||
}
|
||||
|
||||
protected Integer getDomainId(QueryContextReq queryContext, ChatContext chatCtx) {
|
||||
SchemaMapInfo mapInfo = queryContext.getMapInfo();
|
||||
Set<Integer> matchedDomains = mapInfo.getMatchedDomains();
|
||||
Map<Integer, SemanticQuery> domainQueryModes = new HashMap<>();
|
||||
for (Integer matchedDomain : matchedDomains) {
|
||||
domainQueryModes.put(matchedDomain, new LLMSemanticQuery());
|
||||
}
|
||||
List<DomainResolver> domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class,
|
||||
Thread.currentThread().getContextClassLoader());
|
||||
Optional<Integer> domainId = domainResolverList.stream()
|
||||
.map(domainResolver -> domainResolver.resolve(domainQueryModes, queryContext, chatCtx,
|
||||
queryContext.getMapInfo())).filter(d -> d > 0).findFirst();
|
||||
if (domainId.isPresent()) {
|
||||
return domainId.get();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.parser;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
|
||||
import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
|
||||
import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
|
||||
import java.time.LocalDate;
|
||||
import java.util.HashSet;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.Stack;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
|
||||
public class TimeSemanticParser implements SemanticParser {
|
||||
|
||||
private static final Pattern recentPeriodPattern = Pattern.compile(
|
||||
".*(?<periodStr>(近|过去)((?<enNum>\\d+)|(?<zhNum>[一二三四五六七八九十百千万亿]+))个?(?<zhPeriod>[天周月年])).*");
|
||||
|
||||
private int zhNumParse(String zhNumStr) {
|
||||
Stack<Integer> stack = new Stack<>();
|
||||
String numStr = "一二三四五六七八九";
|
||||
String unitStr = "十百千万亿";
|
||||
|
||||
String[] ssArr = zhNumStr.split("");
|
||||
for (String e : ssArr) {
|
||||
int numIndex = numStr.indexOf(e);
|
||||
int unitIndex = unitStr.indexOf(e);
|
||||
if (numIndex != -1) {
|
||||
stack.push(numIndex + 1);
|
||||
} else if (unitIndex != -1) {
|
||||
int unitNum = (int) Math.pow(10, unitIndex + 1);
|
||||
if (stack.isEmpty()) {
|
||||
stack.push(unitNum);
|
||||
} else {
|
||||
stack.push(stack.pop() * unitNum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return stack.stream().mapToInt(s -> s).sum();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||
Matcher m = recentPeriodPattern.matcher(queryContext.getQueryText());
|
||||
if (m.matches()) {
|
||||
int num = 0;
|
||||
String enNum = m.group("enNum");
|
||||
String zhNum = m.group("zhNum");
|
||||
if (enNum != null) {
|
||||
num = Integer.parseInt(enNum);
|
||||
} else if (zhNum != null) {
|
||||
num = zhNumParse(zhNum);
|
||||
}
|
||||
if (num > 0) {
|
||||
DateConf info = new DateConf();
|
||||
String zhPeriod = m.group("zhPeriod");
|
||||
int days;
|
||||
switch (zhPeriod) {
|
||||
case "周":
|
||||
days = 7;
|
||||
info.setPeriod(Constants.WEEK);
|
||||
break;
|
||||
case "月":
|
||||
days = 30;
|
||||
info.setPeriod(Constants.MONTH);
|
||||
break;
|
||||
case "年":
|
||||
days = 365;
|
||||
info.setPeriod(Constants.YEAR);
|
||||
break;
|
||||
default:
|
||||
days = 1;
|
||||
info.setPeriod(Constants.DAY);
|
||||
}
|
||||
days = days * num;
|
||||
info.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
String text = "近" + num + zhPeriod;
|
||||
if (Strings.isNotEmpty(m.group("periodStr"))) {
|
||||
text = m.group("periodStr");
|
||||
}
|
||||
info.setText(text);
|
||||
info.setStartDate(LocalDate.now().minusDays(days).toString());
|
||||
info.setUnit(num);
|
||||
//queryContext.getParseInfo().setDateInfo(info);
|
||||
for (SemanticQuery query : queryContext.getCandidateQueries()) {
|
||||
if (query instanceof MetricSemanticQuery) {
|
||||
query.getParseInfo().setDateInfo(info);
|
||||
}
|
||||
}
|
||||
doParseOnlyTime(queryContext, chatContext, info);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void doParseOnlyTime(QueryContextReq queryContext, ChatContext chatContext, DateConf info) {
|
||||
if (!queryContext.getCandidateQueries().isEmpty() || chatContext.getParseInfo() == null || Objects.isNull(
|
||||
info.getText())) {
|
||||
return;
|
||||
}
|
||||
if (info.getText().equals(queryContext.getQueryText()) && queryContext.getMapInfo().getDomainElementMatches()
|
||||
.isEmpty()
|
||||
) {
|
||||
if (Objects.nonNull(chatContext.getParseInfo().getQueryMode()) && Objects.nonNull(
|
||||
chatContext.getParseInfo().getDomainId()) && chatContext.getParseInfo().getDomainId() > 0) {
|
||||
if (Objects.nonNull(chatContext.getParseInfo().getDateInfo()) && !chatContext.getParseInfo()
|
||||
.getDateInfo().getPeriod().equals(info.getPeriod())) {
|
||||
if (!CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensions())) {
|
||||
String dateField = TimeDimensionEnum.DAY.getName();
|
||||
if (Constants.MONTH.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
|
||||
dateField = TimeDimensionEnum.MONTH.getName();
|
||||
}
|
||||
if (Constants.WEEK.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
|
||||
dateField = TimeDimensionEnum.WEEK.getName();
|
||||
}
|
||||
Set<SchemaItem> dimensions = new HashSet<>();
|
||||
for (SchemaItem schemaItem : chatContext.getParseInfo().getDimensions()) {
|
||||
if (schemaItem.getBizName().equals(dateField)) {
|
||||
continue;
|
||||
}
|
||||
dimensions.add(schemaItem);
|
||||
}
|
||||
chatContext.getParseInfo().setDimensions(dimensions);
|
||||
}
|
||||
}
|
||||
chatContext.getParseInfo().setDateInfo(info);
|
||||
RuleSemanticQuery semanticQuery = RuleSemanticQueryManager.create(
|
||||
chatContext.getParseInfo().getQueryMode());
|
||||
semanticQuery.setParseInfo(chatContext.getParseInfo());
|
||||
queryContext.getCandidateQueries().add(semanticQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class EntityDetail extends EntitySemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "ENTITY_DETAIL";
|
||||
|
||||
public EntityDetail() {
|
||||
super();
|
||||
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
|
||||
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
ContextHelper.addIfEmpty(chatContext.getParseInfo().getDimensionFilters(),
|
||||
parseInfo.getDimensionFilters());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||
import static com.tencent.supersonic.common.constant.Constants.DAY;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||
import com.tencent.supersonic.chat.domain.service.ConfigService;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class EntityListFilter extends EntitySemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
|
||||
private static Long entityListLimit = 200L;
|
||||
|
||||
|
||||
public EntityListFilter() {
|
||||
super();
|
||||
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
|
||||
.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||
parseInfo.setLimit(entityListLimit);
|
||||
this.fillDateEntityFilter(parseInfo);
|
||||
this.addEntityDetailAndOrderByMetric(parseInfo);
|
||||
this.dealNativeQuery(parseInfo, true);
|
||||
}
|
||||
|
||||
|
||||
private void fillDateEntityFilter(SemanticParseInfo semanticParseInfo) {
|
||||
DateConf dateInfo = new DateConf();
|
||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
dateInfo.setUnit(1);
|
||||
dateInfo.setPeriod(DAY);
|
||||
dateInfo.setText(String.format("近1天"));
|
||||
semanticParseInfo.setDateInfo(dateInfo);
|
||||
}
|
||||
|
||||
private void addEntityDetailAndOrderByMetric(SemanticParseInfo semanticParseInfo) {
|
||||
if (semanticParseInfo.getDomainId() > 0L) {
|
||||
ConfigService configService = ContextUtils.getBean(ConfigService.class);
|
||||
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(
|
||||
semanticParseInfo.getDomainId());
|
||||
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
|
||||
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
|
||||
// SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
||||
// EntityRichInfo entity = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
|
||||
Set<SchemaItem> dimensions = new LinkedHashSet();
|
||||
// Set<String> primaryDimensions = this.addPrimaryDimension(entity, dimensions);
|
||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||
Set<Order> orders = new LinkedHashSet();
|
||||
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
|
||||
if (chatDefaultConfig != null) {
|
||||
chatDefaultConfig.getMetrics().stream()
|
||||
.forEach(metric -> {
|
||||
metrics.add(metric);
|
||||
orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER));
|
||||
});
|
||||
chatDefaultConfig.getDimensions().stream()
|
||||
// .filter((m) -> !primaryDimensions.contains(m.getBizName()))
|
||||
.forEach(dimension -> dimensions.add(dimension));
|
||||
|
||||
}
|
||||
|
||||
semanticParseInfo.setDimensions(dimensions);
|
||||
semanticParseInfo.setMetrics(metrics);
|
||||
semanticParseInfo.setOrders(orders);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Set<String> addPrimaryDimension(EntityRichInfo entity, Set<SchemaItem> dimensions) {
|
||||
Set<String> primaryDimensions = new HashSet();
|
||||
DimSchemaResp dimItem = entity.getDimItem();
|
||||
if (Objects.nonNull(entity) && Objects.nonNull(dimItem)) {
|
||||
SchemaItem dimension = new SchemaItem();
|
||||
BeanUtils.copyProperties(dimItem, dimension);
|
||||
dimensions.add(dimension);
|
||||
primaryDimensions.add(dimItem.getBizName());
|
||||
return primaryDimensions;
|
||||
} else {
|
||||
return primaryDimensions;
|
||||
}
|
||||
}
|
||||
|
||||
private void dealNativeQuery(SemanticParseInfo semanticParseInfo, boolean isNativeQuery) {
|
||||
if (Objects.nonNull(semanticParseInfo)) {
|
||||
semanticParseInfo.setNativeQuery(isNativeQuery);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class EntityListTopN extends EntitySemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "ENTITY_LIST_TOPN";
|
||||
|
||||
public EntityListTopN() {
|
||||
super();
|
||||
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
|
||||
.setSupportOrderBy(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class EntityMetricFilter extends EntitySemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "ENTITY_METRIC_FILTER";
|
||||
|
||||
public EntityMetricFilter() {
|
||||
super();
|
||||
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
|
||||
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
|
||||
|
||||
public EntitySemanticQuery() {
|
||||
super();
|
||||
queryMatcher.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import com.tencent.supersonic.chat.application.DomainEntityService;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Slf4j
|
||||
public class LLMSemanticQuery implements SemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "DSL";
|
||||
|
||||
private SemanticParseInfo semanticParse = new SemanticParseInfo();
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultResp execute(User user) {
|
||||
String queryMode = semanticParse.getQueryMode();
|
||||
|
||||
if (semanticParse.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
|
||||
// reach here some error may happen
|
||||
log.error("not find QueryMode");
|
||||
throw new RuntimeException("not find QueryMode");
|
||||
}
|
||||
QueryResultResp queryResponse = new QueryResultResp();
|
||||
QueryResultWithSchemaResp queryResult = semanticLayer.queryBySql(
|
||||
SchemaInfoConverter.convertToQuerySqlReq(semanticParse), user);
|
||||
|
||||
if (queryResult != null) {
|
||||
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
|
||||
}
|
||||
String sql = queryResult == null ? null : queryResult.getSql();
|
||||
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
|
||||
: queryResult.getResultList();
|
||||
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
|
||||
queryResponse.setQuerySql(sql);
|
||||
queryResponse.setQueryResults(resultList);
|
||||
queryResponse.setQueryColumns(columns);
|
||||
queryResponse.setQueryMode(queryMode);
|
||||
|
||||
// add domain info
|
||||
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
|
||||
.getEntityInfo(semanticParse, user);
|
||||
queryResponse.setEntityInfo(entityInfo);
|
||||
return queryResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticParseInfo getParseInfo() {
|
||||
return semanticParse;
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_MOST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.OPTIONAL;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MetricCompare extends MetricSemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "METRIC_COMPARE";
|
||||
public static Pattern intentWordPattern = Pattern.compile("(?i)(比较|对比)");
|
||||
|
||||
public MetricCompare() {
|
||||
super();
|
||||
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 2)
|
||||
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
|
||||
|
||||
queryMatcher.setSupportCompare(true);
|
||||
queryMatcher.setSupportOrderBy(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches, QueryContextReq queryCtx) {
|
||||
if (intentWordPattern.matcher(queryCtx.getQueryText()).find()) {
|
||||
return super.match(candidateElementMatches, queryCtx);
|
||||
} else {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
mergeAppend(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||
addCompareDimension(parseInfo);
|
||||
parseInfo.setBonus(2 * 1.0);
|
||||
}
|
||||
|
||||
private void addCompareDimension(SemanticParseInfo semanticParseInfo) {
|
||||
if (!semanticParseInfo.getDimensionFilters().isEmpty()) {
|
||||
Set<String> dimensions = semanticParseInfo.getDimensions().stream().map(d -> d.getBizName()).collect(
|
||||
Collectors.toSet());
|
||||
log.info("addCompareDimension before [{}]", dimensions);
|
||||
semanticParseInfo.getDimensionFilters().stream().filter(d -> d.getOperator().equals(FilterOperatorEnum.IN))
|
||||
.forEach(
|
||||
d -> {
|
||||
if (!dimensions.contains(d.getBizName())) {
|
||||
SchemaItem schemaItem = new SchemaItem();
|
||||
schemaItem.setBizName(d.getBizName());
|
||||
schemaItem.setId(d.getElementID());
|
||||
semanticParseInfo.getDimensions().add(schemaItem);
|
||||
dimensions.add(d.getBizName());
|
||||
}
|
||||
}
|
||||
);
|
||||
log.info("addCompareDimension after [{}]", dimensions);
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeAppend(Set<Filter> from, Set<Filter> to) {
|
||||
if (!from.isEmpty()) {
|
||||
for (Filter filter : from) {
|
||||
if (FilterOperatorEnum.EQUALS.equals(filter.getOperator()) || FilterOperatorEnum.IN.equals(
|
||||
filter.getOperator())) {
|
||||
Optional<Filter> toAdd = to.stream()
|
||||
.filter(t -> t.getBizName().equalsIgnoreCase(filter.getBizName())).findFirst();
|
||||
if (toAdd.isPresent()) {
|
||||
if (FilterOperatorEnum.EQUALS.equals(toAdd.get().getOperator()) || FilterOperatorEnum.IN.equals(
|
||||
toAdd.get().getOperator())) {
|
||||
Set<Object> vals = new HashSet<>();
|
||||
if (toAdd.get().getOperator().equals(FilterOperatorEnum.IN)) {
|
||||
vals.addAll((List<Object>) (toAdd.get().getValue()));
|
||||
} else {
|
||||
vals.add(toAdd.get().getValue());
|
||||
}
|
||||
if (filter.getOperator().equals(FilterOperatorEnum.IN)) {
|
||||
vals.addAll((List<Object>) (filter.getValue()));
|
||||
} else {
|
||||
vals.add(filter.getValue());
|
||||
}
|
||||
toAdd.get().setValue(new ArrayList<>(vals));
|
||||
toAdd.get().setOperator(FilterOperatorEnum.IN);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
to.add(filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||
|
||||
@Component
|
||||
public class MetricDomain extends MetricSemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "METRIC_DOMAIN";
|
||||
|
||||
public MetricDomain() {
|
||||
super();
|
||||
queryMatcher.addOption(DOMAIN, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||
|
||||
@Component
|
||||
public class MetricFilter extends MetricSemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "METRIC_FILTER";
|
||||
|
||||
public MetricFilter() {
|
||||
super();
|
||||
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
|
||||
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||
|
||||
@Component
|
||||
public class MetricGroupBy extends MetricSemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "METRIC_GROUPBY";
|
||||
|
||||
public MetricGroupBy() {
|
||||
super();
|
||||
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class MetricOrderBy extends RuleSemanticQuery {
|
||||
|
||||
public static String QUERY_MODE = "METRIC_ORDERBY";
|
||||
|
||||
public MetricOrderBy() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryMode() {
|
||||
return QUERY_MODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inheritContext(ChatContext chatContext) {
|
||||
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
|
||||
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||
|
||||
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
|
||||
|
||||
public MetricSemanticQuery() {
|
||||
super();
|
||||
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1);
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import com.tencent.supersonic.chat.application.DomainEntityService;
|
||||
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||
import lombok.ToString;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.*;
|
||||
|
||||
@Slf4j
|
||||
@ToString
|
||||
public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
|
||||
|
||||
protected SemanticParseInfo parseInfo = new SemanticParseInfo();
|
||||
protected QueryMatcher queryMatcher = new QueryMatcher();
|
||||
protected SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
public RuleSemanticQuery() {
|
||||
RuleSemanticQueryManager.register(this);
|
||||
}
|
||||
|
||||
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
|
||||
QueryContextReq queryCtx) {
|
||||
return queryMatcher.match(candidateElementMatches);
|
||||
}
|
||||
|
||||
public abstract void inheritContext(ChatContext chatContext);
|
||||
|
||||
@Override
|
||||
public QueryResultResp execute(User user) {
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
|
||||
if (parseInfo.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
|
||||
// reach here some error may happen
|
||||
log.error("not find QueryMode");
|
||||
throw new RuntimeException("not find QueryMode");
|
||||
}
|
||||
|
||||
List<String> semanticQueryModes = RuleSemanticQueryManager.getSemanticQueryModes();
|
||||
if (!semanticQueryModes.contains(parseInfo.getQueryMode())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
QueryResultResp queryResponse = new QueryResultResp();
|
||||
QueryResultWithSchemaResp queryResult = semanticLayer.queryByStruct(
|
||||
SchemaInfoConverter.convertTo(parseInfo), user);
|
||||
|
||||
|
||||
if (queryResult != null) {
|
||||
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
|
||||
}
|
||||
String sql = queryResult == null ? null : queryResult.getSql();
|
||||
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
|
||||
: queryResult.getResultList();
|
||||
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
|
||||
queryResponse.setQuerySql(sql);
|
||||
queryResponse.setQueryResults(resultList);
|
||||
queryResponse.setQueryColumns(columns);
|
||||
queryResponse.setQueryMode(queryMode);
|
||||
|
||||
// add domain info
|
||||
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
|
||||
.getEntityInfo(parseInfo, user);
|
||||
queryResponse.setEntityInfo(entityInfo);
|
||||
return queryResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticParseInfo getParseInfo() {
|
||||
return parseInfo;
|
||||
}
|
||||
|
||||
public void setParseInfo(SemanticParseInfo parseInfo) {
|
||||
this.parseInfo = parseInfo;
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package com.tencent.supersonic.chat.application.query;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* RuleSemanticQueryManager
|
||||
*/
|
||||
public class RuleSemanticQueryManager {
|
||||
|
||||
private static Map<String, RuleSemanticQuery> semanticQueryMap = new ConcurrentHashMap<>();
|
||||
|
||||
public static RuleSemanticQuery create(String queryMode) {
|
||||
RuleSemanticQuery semanticQuery = semanticQueryMap.get(queryMode);
|
||||
if (Objects.isNull(semanticQuery)) {
|
||||
throw new RuntimeException("no supported queryMode :" + queryMode);
|
||||
}
|
||||
try {
|
||||
return semanticQuery.getClass().getDeclaredConstructor().newInstance();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("no supported queryMode :" + queryMode);
|
||||
}
|
||||
}
|
||||
|
||||
public static void register(RuleSemanticQuery query) {
|
||||
semanticQueryMap.put(query.getQueryMode(), query);
|
||||
}
|
||||
|
||||
public static List<RuleSemanticQuery> getSemanticQueries() {
|
||||
return new ArrayList<>(semanticQueryMap.values());
|
||||
}
|
||||
|
||||
public static List<String> getSemanticQueryModes() {
|
||||
return new ArrayList<>(semanticQueryMap.keySet());
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.util.RecordInfo;
|
||||
import java.util.List;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class ChatConfig {
|
||||
@@ -27,6 +29,8 @@ public class ChatConfig {
|
||||
*/
|
||||
private ChatAggConfig chatAggConfig;
|
||||
|
||||
private List<RecommendedQuestion> recommendedQuestions;
|
||||
|
||||
/**
|
||||
* available status
|
||||
*/
|
||||
@@ -37,4 +41,4 @@ public class ChatConfig {
|
||||
*/
|
||||
private RecordInfo recordInfo;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.enums.StatusEnum;
|
||||
|
||||
import java.util.List;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* extended information command about domain
|
||||
*/
|
||||
@@ -26,9 +27,15 @@ public class ChatConfigBaseReq {
|
||||
*/
|
||||
private ChatAggConfig chatAggConfig;
|
||||
|
||||
|
||||
/**
|
||||
* the recommended questions about the domain
|
||||
*/
|
||||
private List<RecommendedQuestion> recommendedQuestions;
|
||||
|
||||
/**
|
||||
* available status
|
||||
*/
|
||||
private StatusEnum status;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
|
||||
import com.tencent.supersonic.common.enums.StatusEnum;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@@ -15,6 +19,8 @@ public class ChatConfigResp {
|
||||
|
||||
private ChatAggConfig chatAggConfig;
|
||||
|
||||
private List<RecommendedQuestion> recommendedQuestions;
|
||||
|
||||
/**
|
||||
* available status
|
||||
*/
|
||||
@@ -1,12 +1,14 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.enums.StatusEnum;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ChatConfigRichResp {
|
||||
public class ChatConfigRich {
|
||||
|
||||
private Long id;
|
||||
|
||||
@@ -19,6 +21,8 @@ public class ChatConfigRichResp {
|
||||
|
||||
private ChatDetailRichConfig chatDetailRichConfig;
|
||||
|
||||
private List<RecommendedQuestion> recommendedQuestions;
|
||||
|
||||
/**
|
||||
* available status
|
||||
*/
|
||||
@@ -28,4 +32,4 @@ public class ChatConfigRichResp {
|
||||
private String updatedBy;
|
||||
private Date createdAt;
|
||||
private Date updatedAt;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
@@ -10,8 +10,9 @@ import java.util.List;
|
||||
@Data
|
||||
public class ChatDefaultRichConfig {
|
||||
|
||||
private List<SchemaItem> dimensions;
|
||||
private List<SchemaItem> metrics;
|
||||
private List<SchemaElement> dimensions;
|
||||
private List<SchemaElement> metrics;
|
||||
|
||||
|
||||
/**
|
||||
* default time span unit
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class EntityInternalDetail {
|
||||
|
||||
List<DimSchemaResp> dimensionList;
|
||||
List<MetricSchemaResp> metricList;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class EntityRichInfo {
|
||||
/**
|
||||
* entity alias
|
||||
*/
|
||||
private List<String> names;
|
||||
|
||||
private SchemaElement dimItem;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@Data
|
||||
public class FunctionCallConfig {
|
||||
@Value("${functionCall.url:}")
|
||||
private String url;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
import com.tencent.supersonic.common.enums.TypeEnums;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
|
||||
import java.util.List;
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
||||
import lombok.Data;
|
||||
@@ -35,4 +34,4 @@ public class KnowledgeInfo {
|
||||
private KnowledgeAdvancedConfig knowledgeAdvancedConfig;
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.config;
|
||||
package com.tencent.supersonic.chat.config;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
@@ -1,17 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import java.util.Date;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ChatQueryVO {
|
||||
|
||||
private Long questionId;
|
||||
private Date createTime;
|
||||
private Long chatId;
|
||||
private Integer score;
|
||||
private String feedback;
|
||||
private String queryText;
|
||||
private QueryResultResp queryResponse;
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Data;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
/**
|
||||
* DomainInfos
|
||||
*/
|
||||
@Data
|
||||
public class DomainInfos implements Serializable {
|
||||
|
||||
private List<ItemDO> domains = new ArrayList<>();
|
||||
private List<ItemDO> dimensions = new ArrayList<>();
|
||||
private List<ItemDO> metrics = new ArrayList<>();
|
||||
private List<ItemDO> entities = new ArrayList<>();
|
||||
|
||||
public Map<Integer, String> getDomainToName() {
|
||||
if (CollectionUtils.isEmpty(domains)) {
|
||||
return new HashMap();
|
||||
}
|
||||
return domains.stream().collect(
|
||||
Collectors.toMap(ItemDO::getDomain, ItemDO::getName, (value1, value2) -> value2)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class LLMReq {
|
||||
|
||||
private String queryText;
|
||||
private LLMSchema schema;
|
||||
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class LLMSchema {
|
||||
|
||||
private String domainName;
|
||||
|
||||
private List<String> fieldNameList;
|
||||
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class PageQueryInfoReq {
|
||||
|
||||
private int current;
|
||||
|
||||
private int pageSize;
|
||||
|
||||
private String userName;
|
||||
|
||||
public int getPageSize() {
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
public void setPageSize(int pageSize) {
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
public int getCurrent() {
|
||||
return current;
|
||||
}
|
||||
|
||||
public void setCurrent(int current) {
|
||||
this.current = current;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return userName;
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
this.userName = userName;
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class QueryData {
|
||||
|
||||
Long domainId = 0L;
|
||||
Set<SchemaItem> metrics = new HashSet<>();
|
||||
Set<SchemaItem> dimensions = new HashSet<>();
|
||||
Set<Filter> dimensionFilters = new HashSet<>();
|
||||
Set<Filter> metricFilters = new HashSet<>();
|
||||
private Set<Order> orders = new HashSet<>();
|
||||
private DateConf dateInfo;
|
||||
private Long limit;
|
||||
private Boolean nativeQuery = false;
|
||||
|
||||
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Data
|
||||
@Setter
|
||||
@Getter
|
||||
public class RecommendResponse {
|
||||
|
||||
private List<Item> dimensions;
|
||||
private List<Item> metrics;
|
||||
|
||||
|
||||
public static class Item implements Serializable {
|
||||
|
||||
private Integer domain;
|
||||
private String name;
|
||||
private String bizName;
|
||||
|
||||
public Integer getDomain() {
|
||||
return domain;
|
||||
}
|
||||
|
||||
public void setDomain(Integer domain) {
|
||||
this.domain = domain;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getBizName() {
|
||||
return bizName;
|
||||
}
|
||||
|
||||
public void setBizName(String bizName) {
|
||||
this.bizName = bizName;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||
|
||||
public enum SchemaElementOption {
|
||||
REQUIRED,
|
||||
OPTIONAL,
|
||||
UNUSED
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class EntityInternalDetail {
|
||||
|
||||
List<DimSchemaResp> dimensionList;
|
||||
List<MetricSchemaResp> metricList;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class EntityRichInfo {
|
||||
|
||||
// private Long domainId;
|
||||
// private String domainName;
|
||||
// private String domainBizName;
|
||||
|
||||
/**
|
||||
* entity alias
|
||||
*/
|
||||
private List<String> names;
|
||||
|
||||
private DimSchemaResp dimItem;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class NameNature implements Serializable {
|
||||
|
||||
private String name;
|
||||
private List<String> natures;
|
||||
|
||||
public NameNature() {
|
||||
}
|
||||
|
||||
public NameNature(String name, List<String> natures) {
|
||||
this.name = name;
|
||||
this.natures = natures;
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
|
||||
public enum QueryState {
|
||||
NORMAL(0),
|
||||
SEARCH_EXCEPTION(1),
|
||||
EMPTY(2),
|
||||
INVALID(3);
|
||||
|
||||
private int state;
|
||||
|
||||
QueryState(int state) {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
public int getState() {
|
||||
return state;
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Data
|
||||
@Setter
|
||||
@Getter
|
||||
public class SearchResponse {
|
||||
|
||||
private List<SearchResult> searchResults;
|
||||
|
||||
public SearchResponse(List<SearchResult> searchResults) {
|
||||
this.searchResults = searchResults;
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import java.util.Objects;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Data
|
||||
@Setter
|
||||
@Getter
|
||||
public class SearchResult {
|
||||
|
||||
private String recommend;
|
||||
|
||||
private String subRecommend;
|
||||
|
||||
private String domainName;
|
||||
|
||||
private Integer domainId;
|
||||
|
||||
private SchemaElementType schemaElementType;
|
||||
|
||||
private boolean isComplete = true;
|
||||
|
||||
public SearchResult(String recommend, String subRecommend, String className, Integer domainId,
|
||||
SchemaElementType schemaElementType) {
|
||||
this.recommend = recommend;
|
||||
this.subRecommend = subRecommend;
|
||||
this.domainName = className;
|
||||
this.domainId = domainId;
|
||||
this.schemaElementType = schemaElementType;
|
||||
}
|
||||
|
||||
public SearchResult(String recommend, String subRecommend, String className, Integer domainId, boolean isComplete) {
|
||||
this.recommend = recommend;
|
||||
this.subRecommend = subRecommend;
|
||||
this.domainName = className;
|
||||
this.domainId = domainId;
|
||||
this.isComplete = isComplete;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
SearchResult searchResult1 = (SearchResult) o;
|
||||
return Objects.equals(recommend, searchResult1.recommend) && Objects.equals(domainName,
|
||||
searchResult1.domainName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(recommend, domainName);
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.repository;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq;
|
||||
|
||||
public interface ChatQueryRepository {
|
||||
|
||||
PageInfo<ChatQueryVO> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId);
|
||||
|
||||
void createChatQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx);
|
||||
|
||||
ChatQueryDO getLastChatQuery(long chatId);
|
||||
|
||||
int updateChatQuery(ChatQueryDO chatQueryDO);
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.service;
|
||||
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigBaseReq;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigEditReqReq;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigFilter;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ConfigService {
|
||||
|
||||
Long addConfig(ChatConfigBaseReq extendBaseCmd, User user);
|
||||
|
||||
Long editConfig(ChatConfigEditReqReq extendEditCmd, User user);
|
||||
|
||||
List<ChatConfigResp> search(ChatConfigFilter filter, User user);
|
||||
|
||||
ChatConfigRichResp getConfigRichInfo(Long domainId);
|
||||
|
||||
ChatConfigResp fetchConfigByDomainId(Long domainId);
|
||||
|
||||
List<ChatConfigRichResp> getAllChatRichConfig();
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.service;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
|
||||
|
||||
/***
|
||||
* QueryService for query and search
|
||||
*/
|
||||
public interface QueryService {
|
||||
|
||||
QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception;
|
||||
|
||||
SemanticParseInfo queryContext(QueryContextReq queryCtx);
|
||||
|
||||
QueryResultResp executeDirectQuery(QueryData queryData, User user) throws Exception;
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.service;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.RecommendResponse;
|
||||
|
||||
/***
|
||||
* Recommend Service
|
||||
*/
|
||||
public interface RecommendService {
|
||||
|
||||
RecommendResponse recommend(QueryContextReq queryCtx);
|
||||
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.service;
|
||||
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.SearchResult;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* search service
|
||||
*/
|
||||
public interface SearchService {
|
||||
|
||||
List<SearchResult> search(QueryContextReq queryCtx);
|
||||
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.utils;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryMatchInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public class ContextHelper {
|
||||
|
||||
public static Comparator<Map.Entry<Integer, QueryMatchInfo>> DomainStatComparator
|
||||
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
||||
public static Comparator<Map.Entry<SemanticQuery, QueryMatchInfo>> SemanticQueryStatComparator
|
||||
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
||||
/**
|
||||
* similarity desc
|
||||
*/
|
||||
public static Comparator<SchemaElementMatch> schemaElementMatchComparatorBySimilarity
|
||||
= new Comparator<SchemaElementMatch>() {
|
||||
@Override
|
||||
public int compare(SchemaElementMatch o1, SchemaElementMatch o2) {
|
||||
return (int) ((o2.getSimilarity() - o1.getSimilarity()) * 100);
|
||||
}
|
||||
};
|
||||
|
||||
public static void updateDomain(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
if (from != null && from.getDomainId() != null) {
|
||||
to.setDomainId(from.getDomainId());
|
||||
}
|
||||
if (from != null && from.getDomainName() != null) {
|
||||
to.setDomainName(from.getDomainName());
|
||||
}
|
||||
}
|
||||
|
||||
public static void updateEntity(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
if (from != null && from.getEntity() != null && from.getEntity() > 0) {
|
||||
to.setEntity(from.getEntity());
|
||||
}
|
||||
}
|
||||
|
||||
public static void updateSemanticQuery(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
to.setQueryMode(from.getQueryMode());
|
||||
}
|
||||
|
||||
/**
|
||||
* update time if from is not null
|
||||
*
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public static void updateTime(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
if (from != null && from.getDateInfo() != null) {
|
||||
to.setDateInfo(from.getDateInfo());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* update time if time is null and from is not null
|
||||
*
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public static void updateTimeIfEmpty(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
if (from != null && from.getDateInfo() != null && to.getDateInfo() == null) {
|
||||
to.setDateInfo(from.getDateInfo());
|
||||
}
|
||||
}
|
||||
|
||||
public static void updateDomainIfEmpty(SemanticParseInfo from, SemanticParseInfo to) {
|
||||
if (from != null && from.getDomainId() != null && to.getDomainId() == null) {
|
||||
to.setDomainId(from.getDomainId());
|
||||
}
|
||||
|
||||
if (from != null && from.getDomainName() != null && to.getDomainName() == null) {
|
||||
to.setDomainName(from.getDomainName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* add from to list if list is empty and from is not empty
|
||||
*
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public static void addIfEmpty(Set from, Set to) {
|
||||
if (to.isEmpty() && !from.isEmpty()) {
|
||||
to.addAll(from);
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
* append from to list if from is not empty
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public static void appendList(Set from, Set to) {
|
||||
if (!from.isEmpty()) {
|
||||
to.addAll(from);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* update list if from is not empty
|
||||
*
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public static void updateList(Set from, Set to) {
|
||||
if (!from.isEmpty()) {
|
||||
to.clear();
|
||||
to.addAll(from);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* count desc > similarity desc
|
||||
*/
|
||||
public static int domainSchemaElementCountComparator(QueryMatchInfo o1, QueryMatchInfo o2) {
|
||||
int difference = o1.getCount() - o2.getCount();
|
||||
if (difference == 0) {
|
||||
return (int) ((o1.getMaxSimilarity() - o2.getMaxSimilarity()) * 100);
|
||||
}
|
||||
return difference;
|
||||
}
|
||||
|
||||
public static void setEntityId(Long dimensionId, String value, ChatConfigRichResp chaConfigRichDesc,
|
||||
SemanticParseInfo semanticParseInfo) {
|
||||
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
|
||||
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
|
||||
DimSchemaResp dimSchemaResp = chaConfigRichDesc.getChatDetailRichConfig().getEntity().getDimItem();
|
||||
if (Objects.nonNull(dimSchemaResp) && StringUtils.isNumeric(value)) {
|
||||
semanticParseInfo.setEntity(Long.valueOf(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static boolean hasEntityId(ChatContext chatCtx) {
|
||||
if (chatCtx != null && chatCtx.getParseInfo() != null) {
|
||||
return chatCtx.getParseInfo().getEntity() > 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/***
|
||||
* merge Context SchemaElementMatch
|
||||
* @param toSchemaElementMatch
|
||||
* @param elementMatches
|
||||
* @param schemaElementTypes
|
||||
* @param contextSemanticParse
|
||||
*/
|
||||
public static void mergeContextSchemaElementMatch(List<SchemaElementMatch> toSchemaElementMatch,
|
||||
List<SchemaElementMatch> elementMatches, List<SchemaElementType> schemaElementTypes,
|
||||
SemanticParseInfo contextSemanticParse) {
|
||||
|
||||
SchemaElementMatch domainMatch = SchemaElementMatch.builder()
|
||||
.elementType(SchemaElementType.DOMAIN)
|
||||
.elementID(contextSemanticParse.getDomainId().intValue())
|
||||
.similarity(1.0)
|
||||
.word(contextSemanticParse.getDomainName())
|
||||
.detectWord(contextSemanticParse.getDomainName())
|
||||
.build();
|
||||
toSchemaElementMatch.add(domainMatch);
|
||||
|
||||
for (SchemaElementType schemaElementType : schemaElementTypes) {
|
||||
switch (schemaElementType) {
|
||||
case DIMENSION:
|
||||
if (contextSemanticParse.getDimensions().size() > 0) {
|
||||
for (SchemaItem dimension : contextSemanticParse.getDimensions()) {
|
||||
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.DIMENSION,
|
||||
dimension);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case METRIC:
|
||||
if (contextSemanticParse.getMetrics().size() > 0) {
|
||||
for (SchemaItem metric : contextSemanticParse.getMetrics()) {
|
||||
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.METRIC,
|
||||
metric);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case VALUE:
|
||||
if (contextSemanticParse.getDimensionFilters().size() > 0) {
|
||||
for (Filter chatFilter : contextSemanticParse.getDimensionFilters()) {
|
||||
if (!isInSchemaElementMatchList(elementMatches, SchemaElementType.VALUE,
|
||||
chatFilter.getValue().toString())) {
|
||||
List<String> values = new ArrayList<>();
|
||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.IN)) {
|
||||
values.addAll((List<String>) chatFilter.getValue());
|
||||
} else {
|
||||
values.add(chatFilter.getValue().toString());
|
||||
}
|
||||
for (String value : values) {
|
||||
toSchemaElementMatch.add(
|
||||
getSchemaElementMatchByContext(chatFilter.getElementID().intValue(),
|
||||
value, SchemaElementType.VALUE));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* is that SchemaElementType and word in SchemaElementMatch list
|
||||
*
|
||||
* @param elementMatches
|
||||
* @param schemaElementType
|
||||
* @param word
|
||||
* @return
|
||||
*/
|
||||
private static boolean isInSchemaElementMatchList(List<SchemaElementMatch> elementMatches,
|
||||
SchemaElementType schemaElementType, String word) {
|
||||
if (CollectionUtils.isEmpty(elementMatches)) {
|
||||
return false;
|
||||
}
|
||||
Long num = elementMatches.stream()
|
||||
.filter(element -> element != null && element.getElementType().equals(schemaElementType)).count();
|
||||
return num > 0;
|
||||
}
|
||||
|
||||
private static void addSchemaElementMatch(List<SchemaElementMatch> toAddSchemaElementMatch,
|
||||
List<SchemaElementMatch> elementMatches, SchemaElementType schemaElementType, SchemaItem schemaItem) {
|
||||
if (Objects.isNull(schemaItem) || Objects.isNull(schemaItem.getId()) || Objects.isNull(schemaItem.getName())) {
|
||||
return;
|
||||
}
|
||||
if (!isInSchemaElementMatchList(elementMatches, schemaElementType, schemaItem.getName())) {
|
||||
toAddSchemaElementMatch.add(
|
||||
getSchemaElementMatchByContext(schemaItem.getId().intValue(), schemaItem.getName(),
|
||||
schemaElementType));
|
||||
}
|
||||
}
|
||||
|
||||
private static SchemaElementMatch getSchemaElementMatchByContext(int id, String word,
|
||||
SchemaElementType schemaElementType) {
|
||||
return SchemaElementMatch.builder()
|
||||
.elementID(id)
|
||||
.elementType(schemaElementType)
|
||||
.word(word)
|
||||
.detectWord(word)
|
||||
.similarity(0.5)
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,439 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.utils;
|
||||
|
||||
import static java.time.LocalDate.now;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.chat.application.ConfigServiceImpl;
|
||||
import com.tencent.supersonic.chat.application.parser.DomainResolver;
|
||||
import com.tencent.supersonic.chat.application.query.*;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class DefaultMetricUtils {
|
||||
|
||||
@Autowired
|
||||
private ConfigServiceImpl configService;
|
||||
|
||||
/**
|
||||
* supplementary default metric date dimension
|
||||
*/
|
||||
public void fillDefaultMetric(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||
ChatContext chatContext) {
|
||||
String queryMode = semanticParseInfo.getQueryMode();
|
||||
if (StringUtils.isNotEmpty(queryMode)) {
|
||||
Map<String, RuleSemanticQuery> semanticQuery = RuleSemanticQueryManager.getSemanticQueries().stream().collect(Collectors.toMap(RuleSemanticQuery::getQueryMode, Function.identity()));
|
||||
RuleSemanticQuery ruleSemanticQuery = semanticQuery.get(queryMode);
|
||||
if (semanticParseInfo == null) {
|
||||
return;
|
||||
}
|
||||
// if (!EntityListFilter.QUERY_MODE.equals(queryMode)) {
|
||||
boolean isFillAggDefaultMetricLogic = false;
|
||||
boolean isFillDetailDimensionMetric = false;
|
||||
Integer domainId = queryContext.getDomainId().intValue();
|
||||
ChatDefaultRichConfig chatDefaultConfig = null;
|
||||
Boolean isDetailMode = false;
|
||||
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
|
||||
ChatConfigRichResp chaConfigRichDesc = getChatConfigRichInfo(semanticParseInfo.getDomainId());
|
||||
if (Objects.isNull(chaConfigRichDesc)) {
|
||||
return;
|
||||
}
|
||||
if (ruleSemanticQuery instanceof MetricSemanticQuery) {
|
||||
if (!CollectionUtils.isEmpty(matchedElements)) {
|
||||
long metricCount = matchedElements.stream()
|
||||
.filter(schemaElementMatch -> schemaElementMatch.getElementType()
|
||||
.equals(SchemaElementType.METRIC)).count();
|
||||
|
||||
if (metricCount <= 0) {
|
||||
if (chatContext.getParseInfo() == null
|
||||
|| chatContext.getParseInfo().getMetrics() == null
|
||||
|| chatContext.getParseInfo().getMetrics().size() <= 0) {
|
||||
|
||||
log.info("isFillAggDefaultMetricLogic is true");
|
||||
isFillAggDefaultMetricLogic = true;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) {
|
||||
chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
|
||||
}
|
||||
|
||||
} else if (ruleSemanticQuery instanceof EntitySemanticQuery) {
|
||||
log.info("fillThemeDefaultMetricLogic for empty matchedElements ");
|
||||
isFillDetailDimensionMetric = true;
|
||||
dealNativeQuery(semanticParseInfo, queryContext, true);
|
||||
isDetailMode = true;
|
||||
if (Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())) {
|
||||
chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (isFillAggDefaultMetricLogic) {
|
||||
fillDefaultMetricAggLogic(semanticParseInfo, chaConfigRichDesc, queryContext);
|
||||
}
|
||||
|
||||
if (isFillDetailDimensionMetric) {
|
||||
addEntityDetailDimensionMetric(semanticParseInfo, chaConfigRichDesc, queryContext, chatContext);
|
||||
}
|
||||
|
||||
fillDateDomain(semanticParseInfo, chatContext, chaConfigRichDesc, chatDefaultConfig, isDetailMode);
|
||||
// }
|
||||
defaultQueryMode(semanticParseInfo, queryContext, chatContext);
|
||||
addEntityTopDimension(semanticParseInfo, chaConfigRichDesc);
|
||||
}
|
||||
}
|
||||
|
||||
public void dealNativeQuery(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||
boolean isNativeQuery) {
|
||||
if (Objects.nonNull(queryContext) && Objects.nonNull(semanticParseInfo)) {
|
||||
semanticParseInfo.setNativeQuery(isNativeQuery);
|
||||
}
|
||||
}
|
||||
|
||||
public Set<String> addPrimaryDimension(EntityRichInfo entity, List<SchemaItem> dimensions) {
|
||||
Set<String> primaryDimensions = new HashSet<>();
|
||||
if (Objects.isNull(entity) || Objects.isNull(entity.getDimItem())) {
|
||||
return primaryDimensions;
|
||||
}
|
||||
DimSchemaResp dimItem = entity.getDimItem();
|
||||
SchemaItem dimension = new SchemaItem();
|
||||
BeanUtils.copyProperties(dimItem, dimension);
|
||||
dimensions.add(dimension);
|
||||
primaryDimensions.add(dimItem.getBizName());
|
||||
return primaryDimensions;
|
||||
}
|
||||
|
||||
public void addEntityTopDimension(SemanticParseInfo semanticParseInfo, ChatConfigRichResp chaConfigRichDesc) {
|
||||
if (!semanticParseInfo.getQueryMode().equals(EntityListTopN.QUERY_MODE) || !semanticParseInfo.getDimensions()
|
||||
.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (semanticParseInfo.getDomainId() > 0) {
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
if (chaConfigRichDesc == null) {
|
||||
chaConfigRichDesc = getChatConfigRichInfo(domainId);
|
||||
}
|
||||
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
|
||||
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
|
||||
List<SchemaItem> dimensions = new ArrayList<>();
|
||||
addPrimaryDimension(chaConfigRichDesc.getChatDetailRichConfig().getEntity(), dimensions);
|
||||
semanticParseInfo.setDimensions(new HashSet<>(dimensions));
|
||||
semanticParseInfo.setLimit(1L);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void addEntityDetailDimensionMetric(SemanticParseInfo semanticParseInfo, ChatConfigRichResp chaConfigRichDesc, QueryContextReq queryContext,
|
||||
ChatContext chatCtx) {
|
||||
if (semanticParseInfo.getDomainId() > 0) {
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
|
||||
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null) {
|
||||
if (chaConfigRichDesc.getChatDetailRichConfig().getEntity() == null
|
||||
|| chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig() == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
|
||||
.getMatchedElements(domainId.intValue());
|
||||
if (CollectionUtils.isEmpty(schemaElementMatches)
|
||||
|| schemaElementMatches.stream().filter(s -> SchemaElementType.DIMENSION.equals(s.getElementType())).count() <= 0) {
|
||||
log.info("addEntityDetailDimensionMetric catch");
|
||||
if (CollectionUtils.isEmpty(semanticParseInfo.getDimensions())) {
|
||||
Set<SchemaItem> dimensions = new LinkedHashSet();
|
||||
List<SchemaItem> dimensionsConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig().getDimensions();
|
||||
if (!CollectionUtils.isEmpty(dimensionsConfig)) {
|
||||
dimensionsConfig.stream().forEach(m -> dimensions.add(m));
|
||||
}
|
||||
semanticParseInfo.setDimensions(dimensions);
|
||||
}
|
||||
|
||||
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) {
|
||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||
List<SchemaItem> metricsConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig().getMetrics();
|
||||
if (!CollectionUtils.isEmpty(metricsConfig)) {
|
||||
metricsConfig.stream().forEach(m -> metrics.add(m));
|
||||
}
|
||||
semanticParseInfo.setMetrics(metrics);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void defaultQueryMode(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||
ChatContext chatCtx) {
|
||||
SchemaMapInfo schemaMap = queryContext.getMapInfo();
|
||||
if (StringUtils.isEmpty(semanticParseInfo.getQueryMode())) {
|
||||
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) {
|
||||
//
|
||||
Long domain = chatCtx.getParseInfo().getDomainId();
|
||||
String queryMode = chatCtx.getParseInfo().getQueryMode();
|
||||
if (!CollectionUtils.isEmpty(schemaMap.getMatchedDomains()) && schemaMap.getMatchedDomains()
|
||||
.contains(domain.intValue())) {
|
||||
List<SchemaElementMatch> elementMatches = schemaMap.getMatchedElements(domain.intValue());
|
||||
Long filterNUm = elementMatches.stream()
|
||||
.filter(e -> e.getElementType().equals(SchemaElementType.VALUE) || e.getElementType()
|
||||
.equals(SchemaElementType.ID)).count();
|
||||
Long dimensionNUm = elementMatches.stream()
|
||||
.filter(e -> e.getElementType().equals(SchemaElementType.DIMENSION)).count();
|
||||
Long metricrNUm = elementMatches.stream()
|
||||
.filter(e -> e.getElementType().equals(SchemaElementType.METRIC)).count();
|
||||
if (filterNUm > 0 && dimensionNUm > 0 && metricrNUm > 0) {
|
||||
// default as entity detail queryMode
|
||||
log.info("defaultQueryMode [{}]", EntityDetail.QUERY_MODE);
|
||||
semanticParseInfo.setQueryMode(EntityDetail.QUERY_MODE);
|
||||
semanticParseInfo.setDomainId(domain);
|
||||
return;
|
||||
}
|
||||
Long entityNUm = elementMatches.stream()
|
||||
.filter(e -> e.getElementType().equals(SchemaElementType.ENTITY)).count();
|
||||
if (filterNUm <= 0 && dimensionNUm <= 0 && entityNUm <= 0) {
|
||||
// default as metric domain
|
||||
if (metricrNUm > 0 || MetricDomain.QUERY_MODE.equals(queryMode)) {
|
||||
// default as entity detail queryMode
|
||||
log.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
||||
semanticParseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
||||
semanticParseInfo.setDomainId(domain);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (CollectionUtils.isEmpty(schemaMap.getMatchedDomains()) && semanticParseInfo != null
|
||||
&& semanticParseInfo.getDateInfo() != null) {
|
||||
// only query time
|
||||
if (MetricDomain.QUERY_MODE.equals(queryMode)) {
|
||||
// METRIC_DOMAIN context
|
||||
log.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
||||
semanticParseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
||||
semanticParseInfo.setDomainId(domain);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void fillParseInfo(SemanticQuery query, Long domainId, String domainName) {
|
||||
SemanticParseInfo parseInfo = query.getParseInfo();
|
||||
parseInfo.setDomainId(domainId);
|
||||
parseInfo.setDomainName(domainName);
|
||||
parseInfo.setQueryMode(query.getQueryMode());
|
||||
|
||||
|
||||
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(parseInfo.getDomainId(), true);
|
||||
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(parseInfo.getDomainId());
|
||||
Map<Long, DimSchemaResp> dimensionDescMap = domainSchemaDesc.getDimensions().stream()
|
||||
.collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
|
||||
Map<Long, MetricSchemaResp> metricDescMap = domainSchemaDesc.getMetrics().stream()
|
||||
.collect(Collectors.toMap(MetricSchemaResp::getId, Function.identity()));
|
||||
Map<Long, List<SchemaElementMatch>> dim2Values = new HashMap<>();
|
||||
|
||||
List<SchemaElementMatch> elementMatches = query.getParseInfo().getElementMatches();
|
||||
for (SchemaElementMatch schemaMatch : elementMatches) {
|
||||
Long elementID = Long.valueOf(schemaMatch.getElementID());
|
||||
switch (schemaMatch.getElementType()) {
|
||||
case ID:
|
||||
case VALUE:
|
||||
if (dimensionDescMap.containsKey(elementID)) {
|
||||
if (dim2Values.containsKey(elementID)) {
|
||||
dim2Values.get(elementID).add(schemaMatch);
|
||||
} else {
|
||||
dim2Values.put(elementID, new ArrayList<>(Arrays.asList(schemaMatch)));
|
||||
}
|
||||
}
|
||||
break;
|
||||
case DIMENSION:
|
||||
DimSchemaResp dimensionDesc = dimensionDescMap.get(elementID);
|
||||
if (dimensionDesc != null) {
|
||||
SchemaItem dimensionParseInfo = new SchemaItem();
|
||||
dimensionParseInfo.setBizName(dimensionDesc.getBizName());
|
||||
dimensionParseInfo.setName(dimensionDesc.getName());
|
||||
dimensionParseInfo.setId(dimensionDesc.getId());
|
||||
parseInfo.getDimensions().add(dimensionParseInfo);
|
||||
}
|
||||
break;
|
||||
case METRIC:
|
||||
MetricSchemaResp metricDesc = metricDescMap.get(elementID);
|
||||
if (metricDesc != null) {
|
||||
SchemaItem metricItem = new SchemaItem();
|
||||
metricItem.setBizName(metricDesc.getBizName());
|
||||
metricItem.setName(metricDesc.getName());
|
||||
metricItem.setId(metricDesc.getId());
|
||||
metricItem.setCreatedAt(null);
|
||||
metricItem.setUpdatedAt(null);
|
||||
parseInfo.getMetrics().add(metricItem);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
if (!dim2Values.isEmpty()) {
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dim2Values.entrySet()) {
|
||||
DimSchemaResp dimensionDesc = dimensionDescMap.get(entry.getKey());
|
||||
if (entry.getValue().size() == 1) {
|
||||
SchemaElementMatch schemaMatch = entry.getValue().get(0);
|
||||
Filter dimensionFilter = new Filter();
|
||||
dimensionFilter.setValue(schemaMatch.getWord());
|
||||
dimensionFilter.setBizName(dimensionDesc.getBizName());
|
||||
dimensionFilter.setName(dimensionDesc.getName());
|
||||
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
|
||||
dimensionFilter.setElementID(Long.valueOf(schemaMatch.getElementID()));
|
||||
parseInfo.getDimensionFilters().add(dimensionFilter);
|
||||
ContextHelper.setEntityId(entry.getKey(), schemaMatch.getWord(), chaConfigRichDesc,
|
||||
parseInfo);
|
||||
} else {
|
||||
Filter dimensionFilter = new Filter();
|
||||
List<String> vals = new ArrayList<>();
|
||||
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
|
||||
dimensionFilter.setValue(vals);
|
||||
dimensionFilter.setBizName(dimensionDesc.getBizName());
|
||||
dimensionFilter.setName(dimensionDesc.getName());
|
||||
dimensionFilter.setOperator(FilterOperatorEnum.IN);
|
||||
dimensionFilter.setElementID(entry.getKey());
|
||||
parseInfo.getDimensionFilters().add(dimensionFilter);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void fillDateDomain(SemanticParseInfo parseInfo, ChatContext chatCtx, ChatConfigRichResp chaConfigRichDesc,
|
||||
ChatDefaultRichConfig chatDefaultConfig, Boolean isDetailMode) {
|
||||
//SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
||||
|
||||
if (parseInfo == null || parseInfo.getDateInfo() == null) {
|
||||
DomainResolver selectStrategy = ComponentFactory.getDomainResolver();
|
||||
boolean isUpdateTime = false;
|
||||
if (selectStrategy.isDomainSwitch(chatCtx, parseInfo)) {
|
||||
isUpdateTime = true;
|
||||
}
|
||||
if (chatCtx.getParseInfo() == null
|
||||
|| chatCtx.getParseInfo().getDateInfo() == null) {
|
||||
isUpdateTime = true;
|
||||
}
|
||||
if (isUpdateTime && parseInfo != null && parseInfo.getDomainId() > 0) {
|
||||
|
||||
fillThemeDefaultTime(chaConfigRichDesc, parseInfo, chatDefaultConfig, isDetailMode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void fillDefaultMetricAggLogic(SemanticParseInfo semanticParseInfo, ChatConfigRichResp chaConfigRichDesc, QueryContextReq queryContext) {
|
||||
//SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
||||
|
||||
if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatAggRichConfig())
|
||||
|| Objects.isNull(chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig())
|
||||
|| CollectionUtils.isEmpty(chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig().getMetrics())) {
|
||||
log.info("there is no defaultMetricIds info");
|
||||
return;
|
||||
}
|
||||
|
||||
if (queryContext.getMapInfo() == null || !queryContext.getMapInfo().getMatchedDomains()
|
||||
.contains(chaConfigRichDesc.getDomainId().intValue())) {
|
||||
return;
|
||||
}
|
||||
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
|
||||
.getMatchedElements(chaConfigRichDesc.getDomainId().intValue());
|
||||
long metricNum = schemaElementMatches.stream().filter(e -> e.getElementType().equals(SchemaElementType.METRIC))
|
||||
.count();
|
||||
long dimensionNum = schemaElementMatches.stream()
|
||||
.filter(e -> e.getElementType().equals(SchemaElementType.DIMENSION)).count();
|
||||
if (metricNum <= 0 && dimensionNum <= 0) {
|
||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||
chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig().getMetrics().stream().forEach(metric -> {
|
||||
SchemaItem metricTmp = new SchemaItem();
|
||||
metricTmp.setId(metric.getId());
|
||||
metricTmp.setBizName(metric.getBizName());
|
||||
metrics.add(metricTmp);
|
||||
});
|
||||
semanticParseInfo.setMetrics(metrics);
|
||||
}
|
||||
|
||||
if (Objects.isNull(semanticParseInfo.getDateInfo()) || Objects.isNull(
|
||||
semanticParseInfo.getDateInfo().getDateMode())) {
|
||||
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
|
||||
DateConf dateInfo = new DateConf();
|
||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
dateInfo.setUnit(chatDefaultConfig.getUnit());
|
||||
dateInfo.setPeriod(chatDefaultConfig.getPeriod());
|
||||
semanticParseInfo.setDateInfo(dateInfo);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void fillThemeDefaultTime(ChatConfigRichResp chaConfigRichDesc, SemanticParseInfo semanticParseInfo, ChatDefaultRichConfig chatDefaultConfig, Boolean isDetailMode) {
|
||||
if (!Objects.isNull(semanticParseInfo.getDateInfo()) && !Objects.isNull(
|
||||
semanticParseInfo.getDateInfo().getDateMode())) {
|
||||
return;
|
||||
}
|
||||
if (chaConfigRichDesc == null) {
|
||||
chaConfigRichDesc = getChatConfigRichInfo(semanticParseInfo.getDomainId());
|
||||
}
|
||||
if (!Objects.isNull(chaConfigRichDesc) && Objects.nonNull(chatDefaultConfig) && !CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
|
||||
DateConf dateInfo = new DateConf();
|
||||
if (isDetailMode) {
|
||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
dateInfo.setUnit(1);
|
||||
dateInfo.setPeriod(Constants.DAY);
|
||||
} else {
|
||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||
dateInfo.setUnit(chatDefaultConfig.getUnit());
|
||||
dateInfo.setPeriod(chatDefaultConfig.getPeriod());
|
||||
}
|
||||
|
||||
|
||||
semanticParseInfo.setDateInfo(dateInfo);
|
||||
log.info("fillThemeDefaultTime");
|
||||
}
|
||||
}
|
||||
|
||||
public ChatConfigRichResp getChatConfigRichInfo(Long domain) {
|
||||
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain);
|
||||
return chaConfigRichDesc;
|
||||
}
|
||||
|
||||
private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
|
||||
SchemaItem queryMeta = new SchemaItem();
|
||||
queryMeta.setId(metricSchemaDesc.getId());
|
||||
queryMeta.setBizName(metricSchemaDesc.getBizName());
|
||||
queryMeta.setName(metricSchemaDesc.getName());
|
||||
return queryMeta;
|
||||
}
|
||||
|
||||
private SchemaItem getDimension(DimSchemaResp dimSchemaDesc) {
|
||||
SchemaItem queryMeta = new SchemaItem();
|
||||
queryMeta.setId(dimSchemaDesc.getId());
|
||||
queryMeta.setBizName(dimSchemaDesc.getBizName());
|
||||
queryMeta.setName(dimSchemaDesc.getName());
|
||||
return queryMeta;
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.utils;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
|
||||
/***
|
||||
* nature type to schemaType converter
|
||||
*/
|
||||
public class NatureConverter {
|
||||
|
||||
public static SchemaElementType convertTo(String nature) {
|
||||
NatureType natureType = NatureType.getNatureType(nature);
|
||||
SchemaElementType result = null;
|
||||
switch (natureType) {
|
||||
case METRIC:
|
||||
result = SchemaElementType.METRIC;
|
||||
break;
|
||||
case DIMENSION:
|
||||
result = SchemaElementType.DIMENSION;
|
||||
break;
|
||||
case ENTITY:
|
||||
result = SchemaElementType.ENTITY;
|
||||
break;
|
||||
case DOMAIN:
|
||||
result = SchemaElementType.DOMAIN;
|
||||
break;
|
||||
case VALUE:
|
||||
result = SchemaElementType.VALUE;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.utils;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.application.query.MetricCompare;
|
||||
import com.tencent.supersonic.chat.application.query.MetricDomain;
|
||||
import com.tencent.supersonic.chat.application.query.MetricFilter;
|
||||
import com.tencent.supersonic.chat.application.query.MetricGroupBy;
|
||||
import com.tencent.supersonic.chat.application.query.MetricOrderBy;
|
||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||
import com.tencent.supersonic.common.constant.Constants;
|
||||
import com.tencent.supersonic.common.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public class SchemaInfoConverter {
|
||||
|
||||
/***
|
||||
* convert to queryStructReq
|
||||
* @param parseInfo
|
||||
* @return
|
||||
*/
|
||||
public static QueryStructReq convertTo(SemanticParseInfo parseInfo) {
|
||||
QueryStructReq queryStructCmd = new QueryStructReq();
|
||||
queryStructCmd.setDomainId(parseInfo.getDomainId());
|
||||
queryStructCmd.setNativeQuery(parseInfo.getNativeQuery());
|
||||
queryStructCmd.setDateInfo(parseInfo.getDateInfo());
|
||||
|
||||
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
|
||||
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
|
||||
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
|
||||
.collect(Collectors.toList());
|
||||
queryStructCmd.setDimensionFilters(dimensionFilters);
|
||||
|
||||
List<Filter> metricFilters = parseInfo.getMetricFilters().stream()
|
||||
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
|
||||
.collect(Collectors.toList());
|
||||
queryStructCmd.setMetricFilters(metricFilters);
|
||||
|
||||
addDateDimension(parseInfo);
|
||||
List<String> dimensions = parseInfo.getDimensions().stream().map(entry -> entry.getBizName())
|
||||
.collect(Collectors.toList());
|
||||
queryStructCmd.setGroups(dimensions);
|
||||
queryStructCmd.setLimit(parseInfo.getLimit());
|
||||
Set<Order> order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics());
|
||||
queryStructCmd.setOrders(new ArrayList<>(order));
|
||||
queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getMetrics(), parseInfo.getAggType()));
|
||||
return queryStructCmd;
|
||||
}
|
||||
|
||||
|
||||
/***
|
||||
* convert to QuerySqlReq
|
||||
* @param parseInfo
|
||||
* @return
|
||||
*/
|
||||
public static QuerySqlReq convertToQuerySqlReq(SemanticParseInfo parseInfo) {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
Object info = parseInfo.getInfo();
|
||||
if (Objects.nonNull(info)) {
|
||||
querySqlReq.setSql(info.toString());
|
||||
}
|
||||
querySqlReq.setDomainId(parseInfo.getDomainId());
|
||||
return querySqlReq;
|
||||
}
|
||||
|
||||
|
||||
private static List<Aggregator> getAggregatorByMetric(Set<SchemaItem> metrics, AggregateTypeEnum aggregateType) {
|
||||
List<Aggregator> aggregators = new ArrayList<>();
|
||||
String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? ""
|
||||
: aggregateType.name();
|
||||
for (SchemaItem metric : metrics) {
|
||||
aggregators.add(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg)));
|
||||
}
|
||||
return aggregators;
|
||||
}
|
||||
|
||||
private static void addDateDimension(SemanticParseInfo parseInfo) {
|
||||
if (parseInfo != null) {
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
if (parseInfo.getDateInfo() == null) {
|
||||
return;
|
||||
}
|
||||
if (MetricCompare.QUERY_MODE.equals(queryMode)) {
|
||||
if (parseInfo.getAggType() != null && !parseInfo.getAggType().equals(AggregateTypeEnum.NONE)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (parseInfo.getAggType() != null && (parseInfo.getAggType().equals(AggregateTypeEnum.MAX)
|
||||
|| parseInfo.getAggType().equals(AggregateTypeEnum.MIN)) && !CollectionUtils.isEmpty(
|
||||
parseInfo.getDimensions())) {
|
||||
return;
|
||||
}
|
||||
DateConf dateInfo = parseInfo.getDateInfo();
|
||||
String dateField = TimeDimensionEnum.DAY.getName();
|
||||
if (Constants.MONTH.equals(dateInfo.getPeriod())) {
|
||||
dateField = TimeDimensionEnum.MONTH.getName();
|
||||
}
|
||||
if (Constants.WEEK.equals(dateInfo.getPeriod())) {
|
||||
dateField = TimeDimensionEnum.WEEK.getName();
|
||||
}
|
||||
for (SchemaItem dimension : parseInfo.getDimensions()) {
|
||||
if (dimension.getBizName().equalsIgnoreCase(dateField)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
SchemaItem dimension = new SchemaItem();
|
||||
dimension.setBizName(dateField);
|
||||
|
||||
if (MetricDomain.QUERY_MODE.equals(queryMode)
|
||||
|| MetricGroupBy.QUERY_MODE.equals(queryMode)
|
||||
|| MetricFilter.QUERY_MODE.equals(queryMode)
|
||||
|| MetricCompare.QUERY_MODE.equals(queryMode)
|
||||
|| MetricOrderBy.QUERY_MODE.equals(queryMode)) {
|
||||
parseInfo.getDimensions().add(dimension);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static DomainInfos convert(List<DomainSchemaResp> domainSchemaInfos) {
|
||||
DomainInfos result = new DomainInfos();
|
||||
if (CollectionUtils.isEmpty(domainSchemaInfos)) {
|
||||
return result;
|
||||
}
|
||||
for (DomainSchemaResp domainSchemaDesc : domainSchemaInfos) {
|
||||
int domain = Math.toIntExact(domainSchemaDesc.getId());
|
||||
// domain
|
||||
ItemDO domainDO = new ItemDO();
|
||||
domainDO.setDomain(domain);
|
||||
domainDO.setName(domainSchemaDesc.getName());
|
||||
domainDO.setItemId(domain);
|
||||
result.getDomains().add(domainDO);
|
||||
domainDO.setBizName(domainSchemaDesc.getBizName());
|
||||
// entity
|
||||
List<String> entityNames = domainSchemaDesc.getEntityNames();
|
||||
if (!CollectionUtils.isEmpty(entityNames)) {
|
||||
for (String entityName : entityNames) {
|
||||
ItemDO entity = new ItemDO();
|
||||
entity.setDomain(domain);
|
||||
entity.setName(entityName);
|
||||
entity.setItemId(domain);
|
||||
result.getEntities().add(entity);
|
||||
}
|
||||
}
|
||||
// metric
|
||||
for (MetricSchemaResp metric : domainSchemaDesc.getMetrics()) {
|
||||
ItemDO metricDO = new ItemDO();
|
||||
metricDO.setDomain(domain);
|
||||
metricDO.setName(metric.getName());
|
||||
metricDO.setItemId(Math.toIntExact(metric.getId()));
|
||||
metricDO.setUseCnt(metric.getUseCnt());
|
||||
metricDO.setBizName(metric.getBizName());
|
||||
result.getMetrics().add(metricDO);
|
||||
|
||||
String metricAlias = metric.getAlias();
|
||||
if (StringUtils.isNotEmpty(metricAlias)) {
|
||||
ItemDO aliasMetricDO = new ItemDO();
|
||||
BeanUtils.copyProperties(metricDO, aliasMetricDO);
|
||||
aliasMetricDO.setName(metricAlias);
|
||||
result.getMetrics().add(aliasMetricDO);
|
||||
}
|
||||
}
|
||||
// dimension
|
||||
for (DimSchemaResp dimension : domainSchemaDesc.getDimensions()) {
|
||||
ItemDO dimensionDO = new ItemDO();
|
||||
dimensionDO.setDomain(domain);
|
||||
dimensionDO.setName(dimension.getName());
|
||||
dimensionDO.setItemId(Math.toIntExact(dimension.getId()));
|
||||
dimensionDO.setUseCnt(dimension.getUseCnt());
|
||||
dimensionDO.setBizName(dimension.getBizName());
|
||||
result.getDimensions().add(dimensionDO);
|
||||
|
||||
String dimensionAlias = dimension.getAlias();
|
||||
if (StringUtils.isNotEmpty(dimensionAlias)) {
|
||||
ItemDO aliasDimensionDO = new ItemDO();
|
||||
BeanUtils.copyProperties(dimensionDO, aliasDimensionDO);
|
||||
aliasDimensionDO.setName(dimensionAlias);
|
||||
result.getDimensions().add(aliasDimensionDO);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static Set<Order> getOrder(Set<Order> parseOrder, AggregateTypeEnum aggregator, Set<SchemaItem> metrics) {
|
||||
if (!CollectionUtils.isEmpty(parseOrder)) {
|
||||
return parseOrder;
|
||||
}
|
||||
Set<Order> orders = new LinkedHashSet();
|
||||
if (CollectionUtils.isEmpty(metrics)) {
|
||||
return orders;
|
||||
}
|
||||
if ((AggregateTypeEnum.TOPN.equals(aggregator) || AggregateTypeEnum.MAX.equals(aggregator)
|
||||
|| AggregateTypeEnum.MIN.equals(
|
||||
aggregator))) {
|
||||
for (SchemaItem metric : metrics) {
|
||||
Order order = new Order();
|
||||
order.setColumn(metric.getBizName());
|
||||
order.setDirection("desc");
|
||||
orders.add(order);
|
||||
}
|
||||
}
|
||||
return orders;
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package com.tencent.supersonic.chat.domain.utils;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.compress.utils.Lists;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* utils to check current parse info is enough to query result
|
||||
*/
|
||||
@Slf4j
|
||||
public class SemanticSatisfactionChecker {
|
||||
|
||||
private static final double THRESHOLD = 0.8;
|
||||
|
||||
// check all the parse info in candidate
|
||||
public static boolean check(QueryContextReq queryCtx) {
|
||||
for (SemanticQuery query : queryCtx.getCandidateQueries()) {
|
||||
SemanticParseInfo semanticParseInfo = query.getParseInfo();
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
List<SchemaElementMatch> schemaElementMatches = queryCtx.getMapInfo()
|
||||
.getMatchedElements(domainId.intValue());
|
||||
if (check(queryCtx.getQueryText(), semanticParseInfo, schemaElementMatches)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//check single parse info
|
||||
private static boolean check(String text, SemanticParseInfo semanticParseInfo,
|
||||
List<SchemaElementMatch> schemaElementMatches) {
|
||||
if (CollectionUtils.isEmpty(schemaElementMatches)) {
|
||||
return false;
|
||||
}
|
||||
List<String> detectWords = Lists.newArrayList();
|
||||
Map<Integer, String> detectWordMap = schemaElementMatches.stream()
|
||||
.collect(Collectors.toMap(SchemaElementMatch::getElementID, SchemaElementMatch::getDetectWord,
|
||||
(id1, id2) -> id1));
|
||||
// get detect word in text by element id in semantic layer
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
if (domainId != null && domainId > 0) {
|
||||
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
|
||||
if (SchemaElementType.DOMAIN.equals(schemaElementMatch.getElementType())) {
|
||||
detectWords.add(schemaElementMatch.getDetectWord());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Filter filter : semanticParseInfo.getDimensionFilters()) {
|
||||
detectWords.add(
|
||||
detectWordMap.getOrDefault(Optional.ofNullable(filter.getElementID()).orElse(0L).intValue(), ""));
|
||||
}
|
||||
for (SchemaItem schemaItem : semanticParseInfo.getMetrics()) {
|
||||
detectWords.add(
|
||||
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L).intValue(), ""));
|
||||
// only first metric
|
||||
break;
|
||||
}
|
||||
for (SchemaItem schemaItem : semanticParseInfo.getDimensions()) {
|
||||
detectWords.add(
|
||||
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L).intValue(), ""));
|
||||
// only first dimension
|
||||
break;
|
||||
}
|
||||
//compare the length between detect words and query text
|
||||
String detectWordsDistinct = StringUtils.join(new HashSet<>(detectWords), "");
|
||||
int detectWordsLength = detectWordsDistinct.length();
|
||||
int queryTextLength = text.length();
|
||||
double degree = detectWordsLength * 1.0 / queryTextLength;
|
||||
if (degree > THRESHOLD) {
|
||||
log.info("queryMode:{} has satisfied semantic check, degree:{}, detectWords:{}, parse info:{}",
|
||||
semanticParseInfo.getQueryMode(), degree, detectWordsDistinct, semanticParseInfo);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package com.tencent.supersonic.chat.infrastructure.mapper;
|
||||
|
||||
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDOExample;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ChatQueryDOMapper {
|
||||
|
||||
long countByExample(ChatQueryDOExample example);
|
||||
|
||||
int deleteByPrimaryKey(Long questionId);
|
||||
|
||||
|
||||
int insert(ChatQueryDO record);
|
||||
|
||||
int insertSelective(ChatQueryDO record);
|
||||
|
||||
List<ChatQueryDO> selectByExampleWithBLOBs(ChatQueryDOExample example);
|
||||
|
||||
List<ChatQueryDO> selectByExample(ChatQueryDOExample example);
|
||||
|
||||
ChatQueryDO selectByPrimaryKey(Long questionId);
|
||||
|
||||
int updateByPrimaryKeySelective(ChatQueryDO record);
|
||||
|
||||
int updateByPrimaryKeyWithBLOBs(ChatQueryDO record);
|
||||
|
||||
int updateByPrimaryKey(ChatQueryDO record);
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package com.tencent.supersonic.chat.infrastructure.semantic;
|
||||
|
||||
import com.tencent.supersonic.chat.application.ConfigServiceImpl;
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
@Configuration
|
||||
@Data
|
||||
public class DefaultSemanticConfig {
|
||||
|
||||
@Value("${semantic.url.prefix:http://localhost:8081}")
|
||||
private String semanticUrl;
|
||||
|
||||
@Value("${searchByStruct.path:/api/semantic/query/struct}")
|
||||
private String searchByStructPath;
|
||||
|
||||
@Value("${searchByStruct.path:/api/semantic/query/sql}")
|
||||
private String searchBySqlPath;
|
||||
|
||||
@Value("${fetchDomainSchemaPath.path:/api/semantic/schema}")
|
||||
private String fetchDomainSchemaPath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/dimension/page}")
|
||||
private String fetchDimensionPagePath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/metric/page}")
|
||||
private String fetchMetricPagePath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/domain/list}")
|
||||
private String fetchDomainListPath;
|
||||
|
||||
@Value("${fetchDomainList.path:/api/semantic/schema/domain/view/list}")
|
||||
private String fetchDomainViewListPath;
|
||||
|
||||
@Autowired
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
@Autowired
|
||||
private ConfigServiceImpl configService;
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
package com.tencent.supersonic.chat.infrastructure.semantic;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatAggConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatDetailConfig;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.ItemVisibility;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.context.S2ThreadContext;
|
||||
import com.tencent.supersonic.common.util.context.ThreadContext;
|
||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
|
||||
import com.tencent.supersonic.semantic.api.core.response.*;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.core.domain.DimensionService;
|
||||
import com.tencent.supersonic.semantic.core.domain.DomainService;
|
||||
import com.tencent.supersonic.semantic.core.domain.MetricService;
|
||||
import com.tencent.supersonic.semantic.query.domain.SchemaService;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import com.tencent.supersonic.semantic.query.domain.QueryService;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
public class LocalSemanticLayerImpl implements SemanticLayer {
|
||||
|
||||
private static final Cache<String, List<DomainSchemaResp>> domainSchemaCache =
|
||||
CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
|
||||
|
||||
private SchemaService schemaService;
|
||||
|
||||
private S2ThreadContext s2ThreadContext;
|
||||
|
||||
private DomainService domainService;
|
||||
|
||||
private DimensionService dimensionService;
|
||||
|
||||
private MetricService metricService;
|
||||
|
||||
// public LocalSemanticLayerImpl(DomainService domainService){
|
||||
// this.domainService=domainService;
|
||||
// }
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||
deletionDuplicated(queryStructReq);
|
||||
onlyQueryFirstMetric(queryStructReq);
|
||||
try {
|
||||
QueryService queryService = ContextUtils.getBean(QueryService.class);
|
||||
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user);
|
||||
return queryResultWithSchemaResp;
|
||||
} catch (Exception e) {
|
||||
log.info("queryByStruct has an exception:{}", e.toString());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryBySql(QuerySqlReq querySqlReq, User user) {
|
||||
try {
|
||||
QueryService queryService = ContextUtils.getBean(QueryService.class);
|
||||
Object object = queryService.queryBySql(querySqlReq, user);
|
||||
QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object),
|
||||
QueryResultWithSchemaResp.class);
|
||||
return queryResultWithSchemaResp;
|
||||
} catch (Exception e) {
|
||||
log.info("queryByStruct has an exception:{}", e.toString());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public List<DomainSchemaResp> fetchDomainSchemaAll(List<Long> ids) {
|
||||
|
||||
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
|
||||
filter.setDomainIds(ids);
|
||||
User user = new User(1L, "admin", "admin", "admin@email");
|
||||
schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
return schemaService.fetchDomainSchema(filter, user);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public List<DomainSchemaResp> fetchDomainSchema(List<Long> ids, Boolean cacheEnable) {
|
||||
if (cacheEnable) {
|
||||
return domainSchemaCache.get(String.valueOf(ids), () -> {
|
||||
List<DomainSchemaResp> data = fetchDomainSchemaAll(ids);
|
||||
fillEntityNameAndFilterBlackElement(data);
|
||||
return data;
|
||||
});
|
||||
}
|
||||
List<DomainSchemaResp> data = fetchDomainSchemaAll(ids);
|
||||
fillEntityNameAndFilterBlackElement(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
public DomainSchemaResp getDomainSchemaInfo(Long domain, Boolean cacheEnable) {
|
||||
List<Long> ids = new ArrayList<>();
|
||||
ids.add(domain);
|
||||
List<DomainSchemaResp> domainSchemaResps = fetchDomainSchema(ids, cacheEnable);
|
||||
if (!CollectionUtils.isEmpty(domainSchemaResps)) {
|
||||
Optional<DomainSchemaResp> domainSchemaResp = domainSchemaResps.stream()
|
||||
.filter(d -> d.getId().equals(domain)).findFirst();
|
||||
if (domainSchemaResp.isPresent()) {
|
||||
DomainSchemaResp domainSchema = domainSchemaResp.get();
|
||||
return domainSchema;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids) {
|
||||
return fetchDomainSchema(ids, true);
|
||||
}
|
||||
|
||||
public DomainSchemaResp fillEntityNameAndFilterBlackElement(DomainSchemaResp domainSchemaResp) {
|
||||
if (Objects.isNull(domainSchemaResp) || Objects.isNull(domainSchemaResp.getId())) {
|
||||
return domainSchemaResp;
|
||||
}
|
||||
ChatConfigResp chatConfigResp = getConfigBaseInfo(domainSchemaResp.getId());
|
||||
|
||||
// fill entity names
|
||||
fillEntityNamesInfo(domainSchemaResp, chatConfigResp);
|
||||
|
||||
// filter black element
|
||||
filterBlackDim(domainSchemaResp, chatConfigResp);
|
||||
filterBlackMetric(domainSchemaResp, chatConfigResp);
|
||||
return domainSchemaResp;
|
||||
}
|
||||
|
||||
public void fillEntityNameAndFilterBlackElement(List<DomainSchemaResp> domainSchemaRespList) {
|
||||
if (!CollectionUtils.isEmpty(domainSchemaRespList)) {
|
||||
domainSchemaRespList.stream()
|
||||
.forEach(domainSchemaResp -> fillEntityNameAndFilterBlackElement(domainSchemaResp));
|
||||
}
|
||||
}
|
||||
|
||||
private void filterBlackMetric(DomainSchemaResp domainSchemaResp, ChatConfigResp chatConfigResp) {
|
||||
ItemVisibility visibility = generateFinalVisibility(chatConfigResp);
|
||||
if (Objects.nonNull(chatConfigResp) && Objects.nonNull(visibility)
|
||||
&& !CollectionUtils.isEmpty(visibility.getBlackMetricIdList())
|
||||
&& !CollectionUtils.isEmpty(domainSchemaResp.getMetrics())) {
|
||||
List<MetricSchemaResp> metric4Chat = domainSchemaResp.getMetrics().stream()
|
||||
.filter(metric -> !visibility.getBlackMetricIdList().contains(metric.getId()))
|
||||
.collect(Collectors.toList());
|
||||
domainSchemaResp.setMetrics(metric4Chat);
|
||||
}
|
||||
}
|
||||
|
||||
private ItemVisibility generateFinalVisibility(ChatConfigResp chatConfigInfo) {
|
||||
ItemVisibility visibility = new ItemVisibility();
|
||||
|
||||
ChatAggConfig chatAggConfig = chatConfigInfo.getChatAggConfig();
|
||||
ChatDetailConfig chatDetailConfig = chatConfigInfo.getChatDetailConfig();
|
||||
|
||||
// both black is exist
|
||||
if (Objects.nonNull(chatAggConfig) && Objects.nonNull(chatAggConfig.getVisibility())
|
||||
&& Objects.nonNull(chatDetailConfig) && Objects.nonNull(chatDetailConfig.getVisibility())) {
|
||||
List<Long> blackDimIdList = new ArrayList<>();
|
||||
blackDimIdList.addAll(chatAggConfig.getVisibility().getBlackDimIdList());
|
||||
blackDimIdList.retainAll(chatDetailConfig.getVisibility().getBlackDimIdList());
|
||||
List<Long> blackMetricIdList = new ArrayList<>();
|
||||
|
||||
blackMetricIdList.addAll(chatAggConfig.getVisibility().getBlackMetricIdList());
|
||||
blackMetricIdList.retainAll(chatDetailConfig.getVisibility().getBlackMetricIdList());
|
||||
|
||||
visibility.setBlackDimIdList(blackDimIdList);
|
||||
visibility.setBlackMetricIdList(blackMetricIdList);
|
||||
}
|
||||
return visibility;
|
||||
}
|
||||
|
||||
private void filterBlackDim(DomainSchemaResp domainSchemaResp, ChatConfigResp chatConfigResp) {
|
||||
ItemVisibility visibility = generateFinalVisibility(chatConfigResp);
|
||||
if (Objects.nonNull(chatConfigResp) && Objects.nonNull(visibility)
|
||||
&& !CollectionUtils.isEmpty(visibility.getBlackDimIdList())
|
||||
&& !CollectionUtils.isEmpty(domainSchemaResp.getDimensions())) {
|
||||
List<DimSchemaResp> dim4Chat = domainSchemaResp.getDimensions().stream()
|
||||
.filter(dim -> !visibility.getBlackDimIdList().contains(dim.getId()))
|
||||
.collect(Collectors.toList());
|
||||
domainSchemaResp.setDimensions(dim4Chat);
|
||||
}
|
||||
}
|
||||
|
||||
private void fillEntityNamesInfo(DomainSchemaResp domainSchemaResp, ChatConfigResp chatConfigResp) {
|
||||
if (Objects.nonNull(chatConfigResp) && Objects.nonNull(chatConfigResp.getChatDetailConfig())
|
||||
&& Objects.nonNull(chatConfigResp.getChatDetailConfig().getEntity())
|
||||
&& !CollectionUtils.isEmpty(chatConfigResp.getChatDetailConfig().getEntity().getNames())) {
|
||||
domainSchemaResp.setEntityNames(chatConfigResp.getChatDetailConfig().getEntity().getNames());
|
||||
}
|
||||
}
|
||||
|
||||
private void deletionDuplicated(QueryStructReq queryStructReq) {
|
||||
if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) {
|
||||
Set<String> groups = new HashSet<>();
|
||||
groups.addAll(queryStructReq.getGroups());
|
||||
queryStructReq.getGroups().clear();
|
||||
queryStructReq.getGroups().addAll(groups);
|
||||
}
|
||||
}
|
||||
|
||||
private void onlyQueryFirstMetric(QueryStructReq queryStructReq) {
|
||||
if (!CollectionUtils.isEmpty(queryStructReq.getAggregators()) && queryStructReq.getAggregators().size() > 1) {
|
||||
log.info("multi metric in aggregators:{} , only query first one", queryStructReq.getAggregators());
|
||||
queryStructReq.setAggregators(queryStructReq.getAggregators().subList(0, 1));
|
||||
}
|
||||
}
|
||||
|
||||
public ChatConfigResp getConfigBaseInfo(Long domain) {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
return defaultSemanticConfig.getConfigService().fetchConfigByDomainId(domain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForViewer() {
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
ThreadContext threadContext = s2ThreadContext.get();
|
||||
domainService = ContextUtils.getBean(DomainService.class);
|
||||
return domainService.getDomainListForViewer(threadContext.getUserName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForAdmin() {
|
||||
domainService = ContextUtils.getBean(DomainService.class);
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
ThreadContext threadContext = s2ThreadContext.get();
|
||||
return domainService.getDomainListForAdmin(threadContext.getUserName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<DimensionResp> queryDimensionPage(PageDimensionReq pageDimensionCmd) {
|
||||
dimensionService = ContextUtils.getBean(DimensionService.class);
|
||||
return dimensionService.queryDimension(pageDimensionCmd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<MetricResp> queryMetricPage(PageMetricReq pageMetricCmd) {
|
||||
metricService = ContextUtils.getBean(MetricService.class);
|
||||
return metricService.queryMetric(pageMetricCmd);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,381 +0,0 @@
|
||||
package com.tencent.supersonic.chat.infrastructure.semantic;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.gson.Gson;
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.application.ConfigServiceImpl;
|
||||
import com.tencent.supersonic.chat.domain.pojo.config.*;
|
||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.context.S2ThreadContext;
|
||||
import com.tencent.supersonic.common.util.context.ThreadContext;
|
||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
|
||||
import com.tencent.supersonic.semantic.api.core.response.*;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.common.exception.CommonException;
|
||||
import com.tencent.supersonic.common.result.ResultData;
|
||||
import com.tencent.supersonic.common.result.ReturnCode;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
import static com.tencent.supersonic.common.constant.Constants.*;
|
||||
import static com.tencent.supersonic.common.constant.Constants.PAGESIZE_LOWER;
|
||||
|
||||
@Slf4j
|
||||
public class RemoteSemanticLayerImpl implements SemanticLayer {
|
||||
|
||||
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
@Autowired
|
||||
private ConfigServiceImpl configService;
|
||||
|
||||
@Autowired
|
||||
private S2ThreadContext s2ThreadContext;
|
||||
|
||||
@Autowired
|
||||
private AuthenticationConfig authenticationConfig;
|
||||
|
||||
private static final Cache<String, List<DomainSchemaResp>> domainSchemaCache =
|
||||
CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build();
|
||||
private ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
|
||||
};
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||
deletionDuplicated(queryStructReq);
|
||||
onlyQueryFirstMetric(queryStructReq);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
return searchByRestTemplate(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByStructPath(),
|
||||
new Gson().toJson(queryStructReq));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryBySql(QuerySqlReq querySqlReq, User user) {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
|
||||
new Gson().toJson(querySqlReq));
|
||||
}
|
||||
|
||||
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
fillToken(headers);
|
||||
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
|
||||
HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
|
||||
log.info("url:{},searchByRestTemplate:{}", url, entity.getBody());
|
||||
ResultData<QueryResultWithSchemaResp> responseBody;
|
||||
try {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
ResponseEntity<ResultData<QueryResultWithSchemaResp>> responseEntity = defaultSemanticConfig.getRestTemplate()
|
||||
.exchange(requestUrl,
|
||||
HttpMethod.POST, entity, structTypeRef);
|
||||
responseBody = responseEntity.getBody();
|
||||
log.info("ApiResponse<QueryResultWithColumns> responseBody:{}", responseBody);
|
||||
QueryResultWithSchemaResp semanticQuery = new QueryResultWithSchemaResp();
|
||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||
QueryResultWithSchemaResp data = responseBody.getData();
|
||||
semanticQuery.setColumns(data.getColumns());
|
||||
semanticQuery.setResultList(data.getResultList());
|
||||
semanticQuery.setSql(data.getSql());
|
||||
semanticQuery.setQueryAuthorization(data.getQueryAuthorization());
|
||||
return semanticQuery;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("search semantic interface error,url:" + url, e);
|
||||
}
|
||||
throw new CommonException(responseBody.getCode(), responseBody.getMsg());
|
||||
}
|
||||
|
||||
public List<DomainSchemaResp> fetchDomainSchemaAll(List<Long> ids) {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set(UserConstants.INTERNAL, TRUE_LOWER);
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
fillToken(headers);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
|
||||
URI requestUrl = UriComponentsBuilder.fromHttpUrl(
|
||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainSchemaPath()).build()
|
||||
.encode().toUri();
|
||||
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
|
||||
filter.setDomainIds(ids);
|
||||
ParameterizedTypeReference<ResultData<List<DomainSchemaResp>>> responseTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<List<DomainSchemaResp>>>() {
|
||||
};
|
||||
|
||||
HttpEntity<String> entity = new HttpEntity<>(JSON.toJSONString(filter), headers);
|
||||
|
||||
try {
|
||||
ResponseEntity<ResultData<List<DomainSchemaResp>>> responseEntity = defaultSemanticConfig.getRestTemplate()
|
||||
.exchange(requestUrl,
|
||||
HttpMethod.POST, entity, responseTypeRef);
|
||||
ResultData<List<DomainSchemaResp>> responseBody = responseEntity.getBody();
|
||||
log.debug("ApiResponse<fetchDomainSchema> responseBody:{}", responseBody);
|
||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||
List<DomainSchemaResp> data = responseBody.getData();
|
||||
return data;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("fetchDomainSchema interface error", e);
|
||||
}
|
||||
throw new RuntimeException("fetchDomainSchema interface error");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public List<DomainSchemaResp> fetchDomainSchema(List<Long> ids, Boolean cacheEnable) {
|
||||
if (cacheEnable) {
|
||||
return domainSchemaCache.get(String.valueOf(ids), () -> {
|
||||
List<DomainSchemaResp> data = fetchDomainSchemaAll(ids);
|
||||
fillEntityNameAndFilterBlackElement(data);
|
||||
return data;
|
||||
});
|
||||
}
|
||||
List<DomainSchemaResp> data = fetchDomainSchemaAll(ids);
|
||||
fillEntityNameAndFilterBlackElement(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DomainSchemaResp getDomainSchemaInfo(Long domain, Boolean cacheEnable) {
|
||||
List<Long> ids = new ArrayList<>();
|
||||
ids.add(domain);
|
||||
List<DomainSchemaResp> domainSchemaResps = fetchDomainSchema(ids, cacheEnable);
|
||||
if (!CollectionUtils.isEmpty(domainSchemaResps)) {
|
||||
Optional<DomainSchemaResp> domainSchemaResp = domainSchemaResps.stream()
|
||||
.filter(d -> d.getId().equals(domain)).findFirst();
|
||||
if (domainSchemaResp.isPresent()) {
|
||||
DomainSchemaResp domainSchema = domainSchemaResp.get();
|
||||
return domainSchema;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids) {
|
||||
return fetchDomainSchema(ids, true);
|
||||
}
|
||||
|
||||
public DomainSchemaResp fillEntityNameAndFilterBlackElement(DomainSchemaResp domainSchemaResp) {
|
||||
if (Objects.isNull(domainSchemaResp) || Objects.isNull(domainSchemaResp.getId())) {
|
||||
return domainSchemaResp;
|
||||
}
|
||||
ChatConfigResp chaConfigInfo = getConfigBaseInfo(domainSchemaResp.getId());
|
||||
// fill entity names
|
||||
fillEntityNamesInfo(domainSchemaResp, chaConfigInfo);
|
||||
|
||||
// filter black element
|
||||
filterBlackDim(domainSchemaResp, chaConfigInfo);
|
||||
filterBlackMetric(domainSchemaResp, chaConfigInfo);
|
||||
|
||||
return domainSchemaResp;
|
||||
}
|
||||
|
||||
public void fillEntityNameAndFilterBlackElement(List<DomainSchemaResp> domainSchemaRespList) {
|
||||
if (!CollectionUtils.isEmpty(domainSchemaRespList)) {
|
||||
domainSchemaRespList.stream()
|
||||
.forEach(domainSchemaResp -> fillEntityNameAndFilterBlackElement(domainSchemaResp));
|
||||
}
|
||||
}
|
||||
|
||||
private void filterBlackMetric(DomainSchemaResp domainSchemaResp, ChatConfigResp chaConfigInfo) {
|
||||
|
||||
ItemVisibility visibility = generateFinalVisibility(chaConfigInfo);
|
||||
if (Objects.nonNull(chaConfigInfo) && Objects.nonNull(visibility)
|
||||
&& !CollectionUtils.isEmpty(visibility.getBlackMetricIdList())
|
||||
&& !CollectionUtils.isEmpty(domainSchemaResp.getMetrics())) {
|
||||
List<MetricSchemaResp> metric4Chat = domainSchemaResp.getMetrics().stream()
|
||||
.filter(metric -> !visibility.getBlackMetricIdList().contains(metric.getId()))
|
||||
.collect(Collectors.toList());
|
||||
domainSchemaResp.setMetrics(metric4Chat);
|
||||
}
|
||||
}
|
||||
|
||||
private ItemVisibility generateFinalVisibility(ChatConfigResp chatConfigInfo) {
|
||||
ItemVisibility visibility = new ItemVisibility();
|
||||
|
||||
ChatAggConfig chatAggConfig = chatConfigInfo.getChatAggConfig();
|
||||
ChatDetailConfig chatDetailConfig = chatConfigInfo.getChatDetailConfig();
|
||||
|
||||
// both black is exist
|
||||
if (Objects.nonNull(chatAggConfig) && Objects.nonNull(chatAggConfig.getVisibility())
|
||||
&& Objects.nonNull(chatDetailConfig) && Objects.nonNull(chatDetailConfig.getVisibility())) {
|
||||
List<Long> blackDimIdList = new ArrayList<>();
|
||||
blackDimIdList.addAll(chatAggConfig.getVisibility().getBlackDimIdList());
|
||||
blackDimIdList.retainAll(chatDetailConfig.getVisibility().getBlackDimIdList());
|
||||
List<Long> blackMetricIdList = new ArrayList<>();
|
||||
|
||||
blackMetricIdList.addAll(chatAggConfig.getVisibility().getBlackMetricIdList());
|
||||
blackMetricIdList.retainAll(chatDetailConfig.getVisibility().getBlackMetricIdList());
|
||||
|
||||
visibility.setBlackDimIdList(blackDimIdList);
|
||||
visibility.setBlackMetricIdList(blackMetricIdList);
|
||||
}
|
||||
return visibility;
|
||||
}
|
||||
|
||||
private void filterBlackDim(DomainSchemaResp domainSchemaResp, ChatConfigResp chatConfigInfo) {
|
||||
ItemVisibility visibility = generateFinalVisibility(chatConfigInfo);
|
||||
if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(visibility)
|
||||
&& !CollectionUtils.isEmpty(visibility.getBlackDimIdList())
|
||||
&& !CollectionUtils.isEmpty(domainSchemaResp.getDimensions())) {
|
||||
List<DimSchemaResp> dim4Chat = domainSchemaResp.getDimensions().stream()
|
||||
.filter(dim -> !visibility.getBlackDimIdList().contains(dim.getId()))
|
||||
.collect(Collectors.toList());
|
||||
domainSchemaResp.setDimensions(dim4Chat);
|
||||
}
|
||||
}
|
||||
|
||||
private void fillEntityNamesInfo(DomainSchemaResp domainSchemaResp, ChatConfigResp chatConfigInfo) {
|
||||
if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(chatConfigInfo.getChatDetailConfig())
|
||||
&& Objects.nonNull(chatConfigInfo.getChatDetailConfig().getEntity())
|
||||
&& !CollectionUtils.isEmpty(chatConfigInfo.getChatDetailConfig().getEntity().getNames())) {
|
||||
domainSchemaResp.setEntityNames(chatConfigInfo.getChatDetailConfig().getEntity().getNames());
|
||||
}
|
||||
}
|
||||
|
||||
private void deletionDuplicated(QueryStructReq queryStructReq) {
|
||||
if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) {
|
||||
Set<String> groups = new HashSet<>();
|
||||
groups.addAll(queryStructReq.getGroups());
|
||||
queryStructReq.getGroups().clear();
|
||||
queryStructReq.getGroups().addAll(groups);
|
||||
}
|
||||
}
|
||||
|
||||
private void onlyQueryFirstMetric(QueryStructReq queryStructReq) {
|
||||
if (!CollectionUtils.isEmpty(queryStructReq.getAggregators()) && queryStructReq.getAggregators().size() > 1) {
|
||||
log.info("multi metric in aggregators:{} , only query first one", queryStructReq.getAggregators());
|
||||
queryStructReq.setAggregators(queryStructReq.getAggregators().subList(0, 1));
|
||||
}
|
||||
}
|
||||
|
||||
public ChatConfigResp getConfigBaseInfo(Long domain) {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
return defaultSemanticConfig.getConfigService().fetchConfigByDomainId(domain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForViewer() {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object domainDescListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainViewListPath(), null, HttpMethod.GET);
|
||||
List<DomainResp> domainDescList = JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||
return domainDescList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainListForAdmin() {
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object domainDescListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDomainListPath(), null, HttpMethod.GET);
|
||||
List<DomainResp> domainDescList = JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||
return domainDescList;
|
||||
}
|
||||
|
||||
public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
fillToken(headers);
|
||||
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
|
||||
ParameterizedTypeReference<ResultData<Object>> responseTypeRef =
|
||||
new ParameterizedTypeReference<ResultData<Object>>() {
|
||||
};
|
||||
HttpEntity<String> entity = new HttpEntity<>(JsonUtil.toString(bodyJson), headers);
|
||||
try {
|
||||
restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||
ResponseEntity<ResultData<Object>> responseEntity = restTemplate.exchange(requestUrl,
|
||||
httpMethod, entity, responseTypeRef);
|
||||
ResultData<Object> responseBody = responseEntity.getBody();
|
||||
log.debug("ApiResponse<fetchDomainSchema> responseBody:{}", responseBody);
|
||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||
Object data = responseBody.getData();
|
||||
return data;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("fetchDomainSchema interface error", e);
|
||||
}
|
||||
throw new RuntimeException("fetchDomainSchema interface error");
|
||||
}
|
||||
|
||||
public void fillToken(HttpHeaders headers) {
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
|
||||
ThreadContext threadContext = s2ThreadContext.get();
|
||||
if (Objects.nonNull(threadContext) && Strings.isNotEmpty(threadContext.getToken())) {
|
||||
if (Objects.nonNull(authenticationConfig) && Strings.isNotEmpty(
|
||||
authenticationConfig.getTokenHttpHeaderKey())) {
|
||||
headers.set(authenticationConfig.getTokenHttpHeaderKey(), threadContext.getToken());
|
||||
}
|
||||
} else {
|
||||
log.debug("threadContext is null:{}", Objects.isNull(threadContext));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<MetricResp> queryMetricPage(PageMetricReq pageMetricCmd) {
|
||||
String body = JsonUtil.toString(pageMetricCmd);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
log.info("url:{}", defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath());
|
||||
Object dimensionListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchMetricPagePath(), body, HttpMethod.POST);
|
||||
LinkedHashMap map = (LinkedHashMap) dimensionListObject;
|
||||
PageInfo<Object> metricDescObjectPageInfo = generatePageInfo(map);
|
||||
PageInfo<MetricResp> metricDescPageInfo = new PageInfo<>();
|
||||
BeanUtils.copyProperties(metricDescObjectPageInfo, metricDescPageInfo);
|
||||
metricDescPageInfo.setList(metricDescPageInfo.getList());
|
||||
return metricDescPageInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<DimensionResp> queryDimensionPage(PageDimensionReq pageDimensionCmd) {
|
||||
String body = JsonUtil.toString(pageDimensionCmd);
|
||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||
Object dimensionListObject = fetchHttpResult(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchDimensionPagePath(), body, HttpMethod.POST);
|
||||
LinkedHashMap map = (LinkedHashMap) dimensionListObject;
|
||||
PageInfo<Object> dimensionDescObjectPageInfo = generatePageInfo(map);
|
||||
PageInfo<DimensionResp> dimensionDescPageInfo = new PageInfo<>();
|
||||
BeanUtils.copyProperties(dimensionDescObjectPageInfo, dimensionDescPageInfo);
|
||||
dimensionDescPageInfo.setList(dimensionDescPageInfo.getList());
|
||||
return dimensionDescPageInfo;
|
||||
}
|
||||
|
||||
private PageInfo<Object> generatePageInfo(LinkedHashMap map) {
|
||||
PageInfo<Object> pageInfo = new PageInfo<>();
|
||||
pageInfo.setList((List<Object>) map.get(LIST_LOWER));
|
||||
Integer total = (Integer) map.get(TOTAL_LOWER);
|
||||
pageInfo.setTotal(total);
|
||||
Integer pageSize = (Integer) map.get(PAGESIZE_LOWER);
|
||||
pageInfo.setPageSize(pageSize);
|
||||
pageInfo.setPages((int) Math.ceil((double) total / pageSize));
|
||||
return pageInfo;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,16 +1,21 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import java.io.Serializable;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
@Builder
|
||||
public class DomainInfoStat implements Serializable {
|
||||
|
||||
private long domainCount;
|
||||
|
||||
private long metricDomainCount;
|
||||
|
||||
private long dimensionDomainCount;
|
||||
|
||||
private long dimensionValueDomainCount;
|
||||
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import java.io.Serializable;
|
||||
@@ -9,10 +9,10 @@ import lombok.ToString;
|
||||
@ToString
|
||||
public class DomainWithSemanticType implements Serializable {
|
||||
|
||||
private Integer domain;
|
||||
private Long domain;
|
||||
private SchemaElementType semanticType;
|
||||
|
||||
public DomainWithSemanticType(Integer domain, SchemaElementType semanticType) {
|
||||
public DomainWithSemanticType(Long domain, SchemaElementType semanticType) {
|
||||
this.domain = domain;
|
||||
this.semanticType = semanticType;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class FuzzyNameMapper implements SchemaMapper {
|
||||
|
||||
@Override
|
||||
public void map(QueryContext queryContext) {
|
||||
|
||||
log.debug("before db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||
|
||||
List<Term> terms = HanlpHelper.getTerms(queryContext.getRequest().getQueryText());
|
||||
|
||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
||||
|
||||
detectAndAddToSchema(queryContext, terms, semanticSchema.getDimensions(), SchemaElementType.DIMENSION);
|
||||
|
||||
detectAndAddToSchema(queryContext, terms, semanticSchema.getMetrics(), SchemaElementType.METRIC);
|
||||
|
||||
log.debug("after db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||
}
|
||||
|
||||
private void detectAndAddToSchema(QueryContext queryContext, List<Term> terms, List<SchemaElement> domains,
|
||||
SchemaElementType schemaElementType) {
|
||||
try {
|
||||
|
||||
Map<String, Set<SchemaElement>> domainResultSet = getResultSet(queryContext, terms, domains);
|
||||
|
||||
addToSchemaMapInfo(domainResultSet, queryContext.getMapInfo(), schemaElementType);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("detectAndAddToSchema error", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Set<SchemaElement>> getResultSet(QueryContext queryContext, List<Term> terms,
|
||||
List<SchemaElement> domains) {
|
||||
|
||||
String queryText = queryContext.getRequest().getQueryText();
|
||||
|
||||
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||
|
||||
Double metricDimensionThresholdConfig = getThreshold(queryContext, mapperHelper);
|
||||
|
||||
Map<String, Set<SchemaElement>> nameToItems = getNameToItems(domains);
|
||||
|
||||
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
|
||||
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
|
||||
|
||||
Map<String, Set<SchemaElement>> domainResultSet = new HashMap<>();
|
||||
for (Integer startIndex = 0; startIndex <= queryText.length() - 1; ) {
|
||||
for (Integer endIndex = startIndex; endIndex <= queryText.length(); ) {
|
||||
endIndex = mapperHelper.getStepIndex(regOffsetToLength, endIndex);
|
||||
if (endIndex > queryText.length()) {
|
||||
continue;
|
||||
}
|
||||
String detectSegment = queryText.substring(startIndex, endIndex);
|
||||
|
||||
for (Entry<String, Set<SchemaElement>> entry : nameToItems.entrySet()) {
|
||||
String name = entry.getKey();
|
||||
Set<SchemaElement> schemaElements = entry.getValue();
|
||||
if (!name.contains(detectSegment)
|
||||
|| mapperHelper.getSimilarity(detectSegment, name) < metricDimensionThresholdConfig) {
|
||||
continue;
|
||||
}
|
||||
Set<SchemaElement> preSchemaElements = domainResultSet.putIfAbsent(detectSegment,
|
||||
schemaElements);
|
||||
if (Objects.nonNull(preSchemaElements)) {
|
||||
preSchemaElements.addAll(schemaElements);
|
||||
}
|
||||
}
|
||||
}
|
||||
startIndex = mapperHelper.getStepIndex(regOffsetToLength, startIndex);
|
||||
}
|
||||
return domainResultSet;
|
||||
}
|
||||
|
||||
private Double getThreshold(QueryContext queryContext, MapperHelper mapperHelper) {
|
||||
|
||||
Double metricDimensionThresholdConfig = mapperHelper.getMetricDimensionThresholdConfig();
|
||||
Double metricDimensionMinThresholdConfig = mapperHelper.getMetricDimensionMinThresholdConfig();
|
||||
|
||||
Map<Long, List<SchemaElementMatch>> domainElementMatches = queryContext.getMapInfo()
|
||||
.getDomainElementMatches();
|
||||
boolean existElement = domainElementMatches.entrySet().stream()
|
||||
.anyMatch(entry -> entry.getValue().size() >= 1);
|
||||
|
||||
if (!existElement) {
|
||||
double halfThreshold = metricDimensionThresholdConfig / 2;
|
||||
|
||||
metricDimensionThresholdConfig = halfThreshold >= metricDimensionMinThresholdConfig ? halfThreshold
|
||||
: metricDimensionMinThresholdConfig;
|
||||
log.info("domainElementMatches:{} , not exist Element metricDimensionThresholdConfig reduce by half:{}",
|
||||
domainElementMatches, metricDimensionThresholdConfig);
|
||||
}
|
||||
return metricDimensionThresholdConfig;
|
||||
}
|
||||
|
||||
private Map<String, Set<SchemaElement>> getNameToItems(List<SchemaElement> domains) {
|
||||
return domains.stream().collect(
|
||||
Collectors.toMap(SchemaElement::getName, a -> {
|
||||
Set<SchemaElement> result = new HashSet<>();
|
||||
result.add(a);
|
||||
return result;
|
||||
}, (k1, k2) -> {
|
||||
k1.addAll(k2);
|
||||
return k1;
|
||||
}));
|
||||
}
|
||||
|
||||
private void addToSchemaMapInfo(Map<String, Set<SchemaElement>> mapResultRowSet, SchemaMapInfo schemaMap,
|
||||
SchemaElementType schemaElementType) {
|
||||
if (Objects.isNull(mapResultRowSet) || mapResultRowSet.size() <= 0) {
|
||||
return;
|
||||
}
|
||||
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||
|
||||
for (Map.Entry<String, Set<SchemaElement>> entry : mapResultRowSet.entrySet()) {
|
||||
String detectWord = entry.getKey();
|
||||
Set<SchemaElement> schemaElements = entry.getValue();
|
||||
for (SchemaElement schemaElement : schemaElements) {
|
||||
|
||||
List<SchemaElementMatch> elements = schemaMap.getMatchedElements(schemaElement.getDomain());
|
||||
if (CollectionUtils.isEmpty(elements)) {
|
||||
elements = new ArrayList<>();
|
||||
schemaMap.setMatchedElements(schemaElement.getDomain(), elements);
|
||||
}
|
||||
Set<Long> regElementSet = elements.stream()
|
||||
.filter(elementMatch -> schemaElementType.equals(elementMatch.getElement().getType()))
|
||||
.map(elementMatch -> elementMatch.getElement().getId())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
if (regElementSet.contains(schemaElement.getId())) {
|
||||
continue;
|
||||
}
|
||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||
.element(schemaElement)
|
||||
.word(schemaElement.getName())
|
||||
.detectWord(detectWord)
|
||||
.frequency(10000L)
|
||||
.similarity(mapperHelper.getSimilarity(detectWord, schemaElement.getName()))
|
||||
.build();
|
||||
log.info("schemaElementType:{},add to schema, elementMatch {}", schemaElementType, schemaElementMatch);
|
||||
elements.add(schemaElementMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.chat.utils.NatureHelper;
|
||||
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory;
|
||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class HanlpDictMapper implements SchemaMapper {
|
||||
|
||||
@Override
|
||||
public void map(QueryContext queryContext) {
|
||||
|
||||
String queryText = queryContext.getRequest().getQueryText();
|
||||
List<Term> terms = HanlpHelper.getTerms(queryText);
|
||||
|
||||
for (Term term : terms) {
|
||||
log.info("word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency());
|
||||
}
|
||||
Long domainId = queryContext.getRequest().getDomainId();
|
||||
|
||||
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
|
||||
Map<MatchText, List<MapResult>> matchResult = matchStrategy.match(queryText, terms, domainId);
|
||||
|
||||
List<MapResult> matches = getMatches(matchResult);
|
||||
|
||||
HanlpHelper.transLetterOriginal(matches);
|
||||
|
||||
log.info("queryContext:{},matches:{}", queryContext, matches);
|
||||
|
||||
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
|
||||
}
|
||||
|
||||
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap, List<Term> terms) {
|
||||
if (CollectionUtils.isEmpty(mapResults)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Long> wordNatureToFrequency = terms.stream().collect(
|
||||
Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
|
||||
term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
|
||||
|
||||
for (MapResult mapResult : mapResults) {
|
||||
for (String nature : mapResult.getNatures()) {
|
||||
Long domainId = NatureHelper.getDomainId(nature);
|
||||
if (Objects.isNull(domainId)) {
|
||||
continue;
|
||||
}
|
||||
SchemaElementType elementType = NatureHelper.convertToElementType(nature);
|
||||
if (Objects.isNull(elementType)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
|
||||
DomainSchema domainSchema = schemaService.getDomainSchema(domainId);
|
||||
|
||||
BaseWordBuilder baseWordBuilder = WordBuilderFactory.get(DictWordType.getNatureType(nature));
|
||||
Long elementID = baseWordBuilder.getElementID(nature);
|
||||
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
|
||||
|
||||
SchemaElement element = domainSchema.getElement(elementType, elementID);
|
||||
if (element.getType().equals(SchemaElementType.VALUE)) {
|
||||
element.setName(mapResult.getName());
|
||||
}
|
||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||
.element(element)
|
||||
.frequency(frequency)
|
||||
.word(mapResult.getName())
|
||||
.similarity(mapResult.getSimilarity())
|
||||
.detectWord(mapResult.getDetectWord())
|
||||
.build();
|
||||
|
||||
Map<Long, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
|
||||
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domainId,
|
||||
new ArrayList<>());
|
||||
if (schemaElementMatches == null) {
|
||||
schemaElementMatches = domainElementMatches.get(domainId);
|
||||
}
|
||||
schemaElementMatches.add(schemaElementMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<MapResult> getMatches(Map<MatchText, List<MapResult>> matchResult) {
|
||||
List<MapResult> matches = new ArrayList<>();
|
||||
if (Objects.isNull(matchResult)) {
|
||||
return matches;
|
||||
}
|
||||
Optional<List<MapResult>> first = matchResult.entrySet().stream()
|
||||
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
||||
.map(entry -> entry.getValue()).findFirst();
|
||||
|
||||
if (first.isPresent()) {
|
||||
matches = first.get();
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.hankcs.hanlp.algorithm.EditDistance;
|
||||
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
||||
import com.tencent.supersonic.chat.utils.NatureHelper;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -25,6 +25,9 @@ public class MapperHelper {
|
||||
private Integer oneDetectionMaxSize;
|
||||
@Value("${metric.dimension.threshold:0.3}")
|
||||
private Double metricDimensionThresholdConfig;
|
||||
|
||||
@Value("${metric.dimension.min.threshold:0.3}")
|
||||
private Double metricDimensionMinThresholdConfig;
|
||||
@Value("${dimension.value.threshold:0.5}")
|
||||
private Double dimensionValueThresholdConfig;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||
import com.tencent.supersonic.common.nlp.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -11,6 +11,6 @@ import java.util.Map;
|
||||
*/
|
||||
public interface MatchStrategy {
|
||||
|
||||
Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Integer detectDomainId);
|
||||
Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Long detectDomainId);
|
||||
|
||||
}
|
||||
@@ -1,30 +1,19 @@
|
||||
package com.tencent.supersonic.chat.domain.pojo.search;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import java.util.Objects;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@Setter
|
||||
@Getter
|
||||
@ToString
|
||||
@Builder
|
||||
public class MatchText {
|
||||
|
||||
private String regText;
|
||||
|
||||
private String detectSegment;
|
||||
|
||||
public MatchText() {
|
||||
}
|
||||
|
||||
|
||||
public MatchText(String regText, String detectSegment) {
|
||||
this.regText = regText;
|
||||
this.detectSegment = detectSegment;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Slf4j
|
||||
public class QueryFilterMapper implements SchemaMapper {
|
||||
|
||||
@Override
|
||||
public void map(QueryContext queryContext) {
|
||||
QueryRequest queryReq = queryContext.getRequest();
|
||||
Long domainId = queryReq.getDomainId();
|
||||
if (domainId == null || domainId <= 0) {
|
||||
return;
|
||||
}
|
||||
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||
clearOtherSchemaElementMatch(domainId, schemaMapInfo);
|
||||
}
|
||||
|
||||
private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) {
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
|
||||
if (!entry.getKey().equals(domainId)) {
|
||||
entry.getValue().clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||
import com.tencent.supersonic.common.nlp.MapResult;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.Suggester;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
@@ -32,10 +32,11 @@ public class QueryMatchStrategy implements MatchStrategy {
|
||||
private MapperHelper mapperHelper;
|
||||
|
||||
@Override
|
||||
public Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Integer detectDomainId) {
|
||||
if (CollectionUtils.isEmpty(terms) || StringUtils.isEmpty(text)) {
|
||||
public Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Long detectDomainId) {
|
||||
if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
|
||||
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
|
||||
|
||||
@@ -47,13 +48,17 @@ public class QueryMatchStrategy implements MatchStrategy {
|
||||
|
||||
List<MapResult> detects = detect(text, regOffsetToLength, offsetList, detectDomainId);
|
||||
Map<MatchText, List<MapResult>> result = new HashMap<>();
|
||||
MatchText matchText = new MatchText(text, text);
|
||||
|
||||
MatchText matchText = MatchText.builder()
|
||||
.regText(text)
|
||||
.detectSegment(text)
|
||||
.build();
|
||||
result.put(matchText, detects);
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<MapResult> detect(String text, Map<Integer, Integer> regOffsetToLength, List<Integer> offsetList,
|
||||
Integer detectDomainId) {
|
||||
Long detectDomainId) {
|
||||
List<MapResult> results = Lists.newArrayList();
|
||||
|
||||
for (Integer index = 0; index <= text.length() - 1; ) {
|
||||
@@ -75,15 +80,15 @@ public class QueryMatchStrategy implements MatchStrategy {
|
||||
return results;
|
||||
}
|
||||
|
||||
private List<MapResult> detectByStep(String text, Integer detectDomainId, Integer index, Integer i, int offset) {
|
||||
private List<MapResult> detectByStep(String text, Long detectDomainId, Integer index, Integer i, int offset) {
|
||||
String detectSegment = text.substring(index, i);
|
||||
Integer oneDetectionSize = mapperHelper.getOneDetectionSize();
|
||||
// step1. pre search
|
||||
LinkedHashSet<MapResult> mapResults = Suggester.prefixSearch(detectSegment,
|
||||
LinkedHashSet<MapResult> mapResults = SearchService.prefixSearch(detectSegment,
|
||||
mapperHelper.getOneDetectionMaxSize())
|
||||
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
// step2. suffix search
|
||||
LinkedHashSet<MapResult> suffixMapResults = Suggester.suffixSearch(detectSegment, oneDetectionSize)
|
||||
LinkedHashSet<MapResult> suffixMapResults = SearchService.suffixSearch(detectSegment, oneDetectionSize)
|
||||
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
|
||||
mapResults.addAll(suffixMapResults);
|
||||
@@ -99,8 +104,8 @@ public class QueryMatchStrategy implements MatchStrategy {
|
||||
log.debug("detectDomainId:{}, before parseResults:{}", mapResults);
|
||||
mapResults = mapResults.stream().map(entry -> {
|
||||
List<String> natures = entry.getNatures().stream().filter(
|
||||
nature -> nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) || (nature.startsWith(
|
||||
NatureType.NATURE_SPILT))
|
||||
nature -> nature.startsWith(DictWordType.NATURE_SPILT + detectDomainId) || (nature.startsWith(
|
||||
DictWordType.NATURE_SPILT))
|
||||
).collect(Collectors.toList());
|
||||
entry.setNatures(natures);
|
||||
return entry;
|
||||
@@ -1,11 +1,11 @@
|
||||
package com.tencent.supersonic.chat.application.mapper;
|
||||
package com.tencent.supersonic.chat.mapper;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.hankcs.hanlp.seg.common.Term;
|
||||
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||
import com.tencent.supersonic.common.nlp.MapResult;
|
||||
import com.tencent.supersonic.common.nlp.NatureType;
|
||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.Suggester;
|
||||
import com.tencent.supersonic.knowledge.dictionary.MapResult;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -25,10 +25,10 @@ public class SearchMatchStrategy implements MatchStrategy {
|
||||
|
||||
@Override
|
||||
public Map<MatchText, List<MapResult>> match(String text, List<Term> originals,
|
||||
Integer detectDomainId) {
|
||||
Long detectDomainId) {
|
||||
|
||||
Map<Integer, Integer> regOffsetToLength = originals.stream()
|
||||
.filter(entry -> !entry.nature.toString().startsWith(NatureType.NATURE_SPILT))
|
||||
.filter(entry -> !entry.nature.toString().startsWith(DictWordType.NATURE_SPILT))
|
||||
.collect(Collectors.toMap(Term::getOffset, value -> value.word.length(),
|
||||
(value1, value2) -> value2));
|
||||
|
||||
@@ -52,19 +52,19 @@ public class SearchMatchStrategy implements MatchStrategy {
|
||||
String detectSegment = text.substring(detectIndex);
|
||||
|
||||
if (StringUtils.isNotEmpty(detectSegment)) {
|
||||
List<MapResult> mapResults = Suggester.prefixSearch(detectSegment);
|
||||
List<MapResult> suffixMapResults = Suggester.suffixSearch(detectSegment, SEARCH_SIZE);
|
||||
List<MapResult> mapResults = SearchService.prefixSearch(detectSegment);
|
||||
List<MapResult> suffixMapResults = SearchService.suffixSearch(detectSegment, SEARCH_SIZE);
|
||||
mapResults.addAll(suffixMapResults);
|
||||
// remove entity name where search
|
||||
mapResults = mapResults.stream().filter(entry -> {
|
||||
List<String> natures = entry.getNatures().stream()
|
||||
.filter(nature -> !nature.endsWith(NatureType.ENTITY.getType()))
|
||||
.filter(nature -> !nature.endsWith(DictWordType.ENTITY.getType()))
|
||||
.filter(nature -> {
|
||||
if (Objects.isNull(detectDomainId) || detectDomainId <= 0) {
|
||||
return true;
|
||||
}
|
||||
if (nature.startsWith(NatureType.NATURE_SPILT + detectDomainId)
|
||||
&& nature.startsWith(NatureType.NATURE_SPILT)) {
|
||||
if (nature.startsWith(DictWordType.NATURE_SPILT + detectDomainId)
|
||||
&& nature.startsWith(DictWordType.NATURE_SPILT)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -75,7 +75,11 @@ public class SearchMatchStrategy implements MatchStrategy {
|
||||
}
|
||||
return true;
|
||||
}).collect(Collectors.toList());
|
||||
regTextMap.put(new MatchText(regText, detectSegment), mapResults);
|
||||
MatchText matchText = MatchText.builder()
|
||||
.regText(regText)
|
||||
.detectSegment(detectSegment)
|
||||
.build();
|
||||
regTextMap.put(matchText, mapResults);
|
||||
}
|
||||
}
|
||||
);
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.tencent.supersonic.chat.parser;
|
||||
|
||||
public enum ParseMode {
|
||||
|
||||
RULE,
|
||||
EMBEDDING_RECALL,
|
||||
FUNCTION_CALL;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package com.tencent.supersonic.chat.parser;
|
||||
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.compress.utils.Lists;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* This checker can be used by semantic parsers to check if query intent
|
||||
* has already been satisfied by current candidate queries. If so, current
|
||||
* parser could be skipped.
|
||||
*/
|
||||
@Slf4j
|
||||
public class SatisfactionChecker {
|
||||
|
||||
private static final double LONG_TEXT_THRESHOLD = 0.8;
|
||||
private static final double SHORT_TEXT_THRESHOLD = 0.6;
|
||||
private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10;
|
||||
|
||||
public static final double BONUS_THRESHOLD = 100;
|
||||
|
||||
// check all the parse info in candidate
|
||||
public static boolean check(QueryContext queryCtx) {
|
||||
for (SemanticQuery query : queryCtx.getCandidateQueries()) {
|
||||
SemanticParseInfo semanticParseInfo = query.getParseInfo();
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
List<SchemaElementMatch> schemaElementMatches = queryCtx.getMapInfo()
|
||||
.getMatchedElements(domainId);
|
||||
if (check(queryCtx.getRequest().getQueryText(), semanticParseInfo, schemaElementMatches)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//check single parse info
|
||||
private static boolean check(String text, SemanticParseInfo semanticParseInfo,
|
||||
List<SchemaElementMatch> schemaElementMatches) {
|
||||
if (semanticParseInfo.getBonus() != null && semanticParseInfo.getBonus() >= BONUS_THRESHOLD) {
|
||||
return true;
|
||||
}
|
||||
if (CollectionUtils.isEmpty(schemaElementMatches)) {
|
||||
return false;
|
||||
}
|
||||
List<String> detectWords = Lists.newArrayList();
|
||||
Map<Long, String> detectWordMap = schemaElementMatches.stream()
|
||||
.collect(Collectors.toMap(m -> m.getElement().getId(), SchemaElementMatch::getDetectWord,
|
||||
(id1, id2) -> id1));
|
||||
// get detect word in text by element id in semantic layer
|
||||
Long domainId = semanticParseInfo.getDomainId();
|
||||
if (domainId != null && domainId > 0) {
|
||||
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
|
||||
if (SchemaElementType.DOMAIN.equals(schemaElementMatch.getElement().getType())) {
|
||||
detectWords.add(schemaElementMatch.getDetectWord());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
|
||||
if (SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) {
|
||||
detectWords.add(schemaElementMatch.getDetectWord());
|
||||
}
|
||||
}
|
||||
for (SchemaElement schemaItem : semanticParseInfo.getMetrics()) {
|
||||
detectWords.add(
|
||||
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), ""));
|
||||
// only first metric
|
||||
break;
|
||||
}
|
||||
for (SchemaElement schemaItem : semanticParseInfo.getDimensions()) {
|
||||
detectWords.add(
|
||||
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), ""));
|
||||
// only first dimension
|
||||
break;
|
||||
}
|
||||
String dateText = Optional.ofNullable(semanticParseInfo.getDateInfo()).orElse(new DateConf()).getText();
|
||||
if (StringUtils.isNotBlank(dateText) && !dateText.equalsIgnoreCase(Constants.NULL)) {
|
||||
detectWords.add(dateText);
|
||||
}
|
||||
detectWords.removeIf(word -> !text.contains(word));
|
||||
//compare the length between detect words and query text
|
||||
return checkThreshold(text, detectWords, semanticParseInfo);
|
||||
}
|
||||
|
||||
private static boolean checkThreshold(String queryText, List<String> detectWords, SemanticParseInfo semanticParseInfo) {
|
||||
String detectWordsDistinct = StringUtils.join(new HashSet<>(detectWords), "");
|
||||
int detectWordsLength = detectWordsDistinct.length();
|
||||
int queryTextLength = queryText.length();
|
||||
double degree = detectWordsLength * 1.0 / queryTextLength;
|
||||
if (queryTextLength > QUERY_TEXT_LENGTH_THRESHOLD) {
|
||||
if (degree < LONG_TEXT_THRESHOLD) {
|
||||
return false;
|
||||
}
|
||||
} else if (degree < SHORT_TEXT_THRESHOLD) {
|
||||
return false;
|
||||
}
|
||||
log.info("queryMode:{}, degree:{}, detectWords:{}, parse info:{}",
|
||||
semanticParseInfo.getQueryMode(), degree, detectWordsDistinct, semanticParseInfo);
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.tencent.supersonic.chat.parser.embedding;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.config.ChatConfigRich;
|
||||
import com.tencent.supersonic.chat.config.EntityRichInfo;
|
||||
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
|
||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
|
||||
import com.tencent.supersonic.chat.service.ConfigService;
|
||||
import com.tencent.supersonic.chat.service.PluginService;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class EmbeddingBasedParser implements SemanticParser {
|
||||
|
||||
private final static double THRESHOLD = 0.2d;
|
||||
|
||||
@Override
|
||||
public void parse(QueryContext queryContext, ChatContext chatContext) {
|
||||
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
|
||||
if (SatisfactionChecker.check(queryContext) || StringUtils.isBlank(embeddingConfig.getUrl())) {
|
||||
return;
|
||||
}
|
||||
log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext);
|
||||
for (Long domainId : getDomainMatched(queryContext)) {
|
||||
String text = replaceText(queryContext, domainId);
|
||||
List<RecallRetrieval> embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext));
|
||||
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, domainId);
|
||||
if (pluginOptional.isPresent()) {
|
||||
Map<String, RecallRetrieval> embeddingRetrievalMap = embeddingRetrievals.stream()
|
||||
.collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1));
|
||||
Plugin plugin = pluginOptional.get();
|
||||
log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]",
|
||||
text, domainId, plugin.getId(), plugin.getName());
|
||||
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
|
||||
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(queryContext, domainId,
|
||||
plugin, embeddingRetrievalMap);
|
||||
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
|
||||
pluginQuery.setParseInfo(semanticParseInfo);
|
||||
queryContext.getCandidateQueries().add(pluginQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Set<Long> getDomainMatched(QueryContext queryContext) {
|
||||
Long queryDomainId = queryContext.getRequest().getDomainId();
|
||||
if (queryDomainId != null && queryDomainId > 0) {
|
||||
return Sets.newHashSet(queryDomainId);
|
||||
}
|
||||
return queryContext.getMapInfo().getMatchedDomains();
|
||||
}
|
||||
|
||||
private SemanticParseInfo buildSemanticParseInfo(QueryContext queryContext, Long domainId, Plugin plugin,
|
||||
Map<String, RecallRetrieval> embeddingRetrievalMap) {
|
||||
SchemaElement schemaElement = new SchemaElement();
|
||||
schemaElement.setDomain(domainId);
|
||||
schemaElement.setId(domainId);
|
||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||
semanticParseInfo.setDomain(schemaElement);
|
||||
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||
if (Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance()) < THRESHOLD) {
|
||||
semanticParseInfo.setBonus(SatisfactionChecker.BONUS_THRESHOLD);
|
||||
}
|
||||
Map<String, Object> properties = new HashMap<>();
|
||||
properties.put(Constants.CONTEXT, plugin);
|
||||
semanticParseInfo.setProperties(properties);
|
||||
semanticParseInfo.setElementMatches(schemaMapInfo.getMatchedElements(domainId));
|
||||
fillSemanticParseInfo(queryContext, semanticParseInfo);
|
||||
setEntityId(domainId, semanticParseInfo);
|
||||
return semanticParseInfo;
|
||||
}
|
||||
|
||||
private Optional<Long> getEntityElementId(Long domainId) {
|
||||
ConfigService configService = ContextUtils.getBean(ConfigService.class);
|
||||
ChatConfigRich chatConfigRich = configService.getConfigRichInfo(domainId);
|
||||
EntityRichInfo entityRichInfo = chatConfigRich.getChatDetailRichConfig().getEntity();
|
||||
if (entityRichInfo != null) {
|
||||
SchemaElement schemaElement = entityRichInfo.getDimItem();
|
||||
if (schemaElement != null) {
|
||||
return Optional.of(schemaElement.getId());
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private void setEntityId(Long domainId, SemanticParseInfo semanticParseInfo) {
|
||||
Optional<Long> entityElementIdOptional = getEntityElementId(domainId);
|
||||
if (entityElementIdOptional.isPresent()) {
|
||||
Long entityElementId = entityElementIdOptional.get();
|
||||
for (QueryFilter filter : semanticParseInfo.getDimensionFilters()) {
|
||||
if (entityElementId.equals(filter.getElementID())) {
|
||||
String value = String.valueOf(filter.getValue());
|
||||
if (StringUtils.isNumeric(value)) {
|
||||
semanticParseInfo.setEntity(Long.parseLong(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<Plugin> choosePlugin(List<RecallRetrieval> embeddingRetrievals,
|
||||
Long domainId) {
|
||||
if (CollectionUtils.isEmpty(embeddingRetrievals)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
PluginService pluginService = ContextUtils.getBean(PluginService.class);
|
||||
List<Plugin> plugins = pluginService.getPluginList();
|
||||
Map<Long, Plugin> pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p));
|
||||
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
||||
Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId()));
|
||||
if (!CollectionUtils.isEmpty(plugin.getDomainList()) && plugin.getDomainList().contains(domainId)) {
|
||||
return Optional.of(plugin);
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
public List<RecallRetrieval> recallResult(String embeddingText, boolean hasCandidateQuery) {
|
||||
try {
|
||||
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
|
||||
EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText);
|
||||
log.info("embedding result, text:{} embeddingResp:{}", embeddingText, embeddingResp);
|
||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||
if(!CollectionUtils.isEmpty(embeddingRetrievals)){
|
||||
if (hasCandidateQuery) {
|
||||
embeddingRetrievals = embeddingRetrievals.stream()
|
||||
.filter(llmRetrieval -> Double.parseDouble(llmRetrieval.getDistance())<THRESHOLD)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
embeddingRetrievals = embeddingRetrievals.stream().sorted(Comparator.comparingDouble(o ->
|
||||
Math.abs(Double.parseDouble(o.getDistance())))).collect(Collectors.toList());
|
||||
embeddingResp.setRetrieval(embeddingRetrievals);
|
||||
}
|
||||
return embeddingRetrievals;
|
||||
} catch (Exception e) {
|
||||
log.warn("get embedding result error ", e);
|
||||
}
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
|
||||
private boolean hasCandidateQuery(QueryContext queryContext) {
|
||||
return !CollectionUtils.isEmpty(queryContext.getCandidateQueries());
|
||||
}
|
||||
|
||||
private void fillSemanticParseInfo(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||
if (queryContext.getRequest().getQueryFilters() != null) {
|
||||
semanticParseInfo.getDimensionFilters()
|
||||
.addAll(queryContext.getRequest().getQueryFilters().getFilters());
|
||||
}
|
||||
}
|
||||
|
||||
protected String replaceText(QueryContext queryContext, Long domainId) {
|
||||
String text = queryContext.getRequest().getQueryText();
|
||||
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo().getMatchedElements(domainId);
|
||||
if (CollectionUtils.isEmpty(schemaElementMatches)) {
|
||||
return text;
|
||||
}
|
||||
List<SchemaElementMatch> valueSchemaElementMatches = schemaElementMatches.stream()
|
||||
.filter(schemaElementMatch ->
|
||||
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toList());
|
||||
for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) {
|
||||
String detectWord = schemaElementMatch.getDetectWord();
|
||||
text = text.replace(detectWord, "");
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.tencent.supersonic.chat.parser.embedding;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@Data
|
||||
public class EmbeddingConfig {
|
||||
|
||||
@Value("${embedding.url:}")
|
||||
private String url;
|
||||
|
||||
@Value("${embedding.recognize.path:preset_query_retrival}")
|
||||
private String recognizePath;
|
||||
|
||||
@Value("${embedding.delete.path:preset_delete_by_ids}")
|
||||
private String deletePath;
|
||||
|
||||
@Value("${embedding.add.path:preset_query_add}")
|
||||
private String addPath;
|
||||
|
||||
@Value("${embedding.nResult:1}")
|
||||
private String nResult;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package com.tencent.supersonic.chat.parser.embedding;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.tencent.supersonic.chat.api.pojo.*;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.chat.config.ChatConfigRich;
|
||||
import com.tencent.supersonic.chat.parser.function.DomainResolver;
|
||||
import com.tencent.supersonic.chat.service.ConfigService;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Component("EmbeddingEntityResolver")
|
||||
public class EmbeddingEntityResolver {
|
||||
|
||||
private ConfigService configService;
|
||||
|
||||
public EmbeddingEntityResolver(ConfigService configService) {
|
||||
this.configService = configService;
|
||||
}
|
||||
|
||||
public Pair<Long, Long> getDomainEntityId(QueryContext queryCtx, ChatContext chatCtx) {
|
||||
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
|
||||
Long domainId = domainResolver.resolve(queryCtx, chatCtx);
|
||||
ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(domainId);
|
||||
SchemaElement schemaElement = chatConfigRichResp.getChatDetailRichConfig().getEntity().getDimItem();
|
||||
if (schemaElement == null) {
|
||||
return Pair.of(domainId, null);
|
||||
}
|
||||
Long entityId = getEntityValue(domainId, schemaElement.getId(), queryCtx, chatCtx);
|
||||
return Pair.of(domainId, entityId);
|
||||
}
|
||||
|
||||
|
||||
private Long getEntityValue(Long domainId, Long entityElementId, QueryContext queryCtx, ChatContext chatCtx) {
|
||||
Long entityId = null;
|
||||
QueryFilters queryFilters = queryCtx.getRequest().getQueryFilters();
|
||||
if (queryFilters != null) {
|
||||
entityId = getEntityValueFromQueryFilter(queryFilters.getFilters());
|
||||
if (entityId != null) {
|
||||
log.info("get entity id:{} domain id:{} from query filter :{} ", entityId, domainId, queryFilters);
|
||||
return entityId;
|
||||
}
|
||||
}
|
||||
entityId = getEntityValueFromSchemaMapInfo(domainId, queryCtx.getMapInfo(), entityElementId);
|
||||
log.info("get entity id:{} from schema map Info :{} ", entityId, JSONObject.toJSONString(queryCtx.getMapInfo()));
|
||||
if (entityId == null || entityId == 0) {
|
||||
Long entityIdFromChat = getEntityValueFromParseInfo(chatCtx.getParseInfo(), entityElementId);
|
||||
if (entityIdFromChat != null && entityIdFromChat > 0) {
|
||||
entityId = entityIdFromChat;
|
||||
}
|
||||
}
|
||||
return entityId;
|
||||
}
|
||||
|
||||
private Long getEntityValueFromQueryFilter(List<QueryFilter> queryFilters) {
|
||||
if (CollectionUtils.isEmpty(queryFilters)) {
|
||||
return null;
|
||||
}
|
||||
QueryFilter filter = queryFilters.get(0);
|
||||
String value = String.valueOf(filter.getValue());
|
||||
if (StringUtils.isNumeric(value)) {
|
||||
return Long.parseLong(value);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Long getEntityValueFromParseInfo(SemanticParseInfo semanticParseInfo, Long entityElementId) {
|
||||
Set<QueryFilter> filters = semanticParseInfo.getDimensionFilters();
|
||||
if (CollectionUtils.isEmpty(filters)) {
|
||||
return null;
|
||||
}
|
||||
for (QueryFilter filter : filters) {
|
||||
if (entityElementId.equals(filter.getElementID())) {
|
||||
String value = String.valueOf(filter.getValue());
|
||||
if (StringUtils.isNumeric(value)) {
|
||||
return Long.parseLong(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private Long getEntityValueFromSchemaMapInfo(Long domainId, SchemaMapInfo schemaMapInfo, Long entityElementId) {
|
||||
List<SchemaElementMatch> schemaElementMatchList = schemaMapInfo.getMatchedElements(domainId);
|
||||
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
|
||||
return null;
|
||||
}
|
||||
for (SchemaElementMatch schemaElementMatch : schemaElementMatchList) {
|
||||
if (Objects.equals(schemaElementMatch.getElement().getId(), entityElementId)) {
|
||||
if (StringUtils.isNumeric(schemaElementMatch.getWord())) {
|
||||
return Long.parseLong(schemaElementMatch.getWord());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.tencent.supersonic.chat.parser.embedding;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class EmbeddingResp {
|
||||
|
||||
private String query;
|
||||
|
||||
private List<RecallRetrieval> retrieval;
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.tencent.supersonic.chat.parser.embedding;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class RecallRetrieval {
|
||||
|
||||
private String id;
|
||||
|
||||
private String distance;
|
||||
|
||||
private String presetQuery;
|
||||
|
||||
private String presetId;
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user