topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.0 version backend update (#24)

Browse Source 
* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version readme update

---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
SunDean
2023-08-05 22:17:56 +08:00
committed by GitHub
parent 6951eada9d
commit aa0a100a85
184 changed files with 2609 additions and 1238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
chat/core/src/main/java/com/tencent/supersonic/chat/config/AggregatorConfig.java Normal file
View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class AggregatorConfig {
@Value("${metric.aggregator.ratio.enable:true}")
private Boolean enableRatio;
}

24
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggConfig.java
View File

@@ -1,24 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibility visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfig chatDefaultConfig;
}

24
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggRichConfig.java
View File

@@ -1,24 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggRichConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfig chatDefaultConfig;
}

10
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
@@ -22,14 +24,14 @@ public class ChatConfig {
/**
* the chatDetailConfig about the domain
*/
private ChatDetailConfig chatDetailConfig;
private ChatDetailConfigReq chatDetailConfig;
/**
* the chatAggConfig about the domain
*/
private ChatAggConfig chatAggConfig;
private ChatAggConfigReq chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions;
private List<RecommendedQuestionReq> recommendedQuestions;
/**
* available status

41
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigBaseReq.java
View File

@@ -1,41 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.ToString;
import java.util.List;
/**
* extended information command about domain
*/
@Data
@ToString
public class ChatConfigBaseReq {
private Long domainId;
/**
* the chatDetailConfig about the domain
*/
private ChatDetailConfig chatDetailConfig;
/**
* the chatAggConfig about the domain
*/
private ChatAggConfig chatAggConfig;
/**
* the recommended questions about the domain
*/
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
private StatusEnum status;
}

10
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigEditReqReq.java
View File

@@ -1,10 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
@Data
public class ChatConfigEditReqReq extends ChatConfigBaseReq {
private Long id;
}

15
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilter.java
View File

@@ -1,15 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@NoArgsConstructor
@Data
public class ChatConfigFilter {
private Long id;
private Long domainId;
private StatusEnum status = StatusEnum.ONLINE;
}

33
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigResp.java
View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
public class ChatConfigResp {
private Long id;
private Long domainId;
private ChatDetailConfig chatDetailConfig;
private ChatAggConfig chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
private StatusEnum statusEnum;
private String createdBy;
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigRich.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
public class ChatConfigRich {
private Long id;
private Long domainId;
private String domainName;
private String bizName;
private ChatAggRichConfig chatAggRichConfig;
private ChatDetailRichConfig chatDetailRichConfig;
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
private StatusEnum statusEnum;
private String createdBy;
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}

27
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultConfig.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class ChatDefaultConfig {
private List<Long> dimensionIds = new ArrayList<>();
private List<Long> metricIds = new ArrayList<>();
/**
* default time span unit
*/
private Integer unit = 1;
/**
* default time type: day
* DAY, WEEK, MONTH, YEAR
*/
private String period = Constants.DAY;
}

28
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultRichConfig.java
View File

@@ -1,28 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.List;
@Data
public class ChatDefaultRichConfig {
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
/**
* default time span unit
*/
private Integer unit = 1;
/**
* default time type: day
* DAY, WEEK, MONTH, YEAR
*/
private String period = Constants.DAY;
}

29
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailConfig.java
View File

@@ -1,29 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibility visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfig chatDefaultConfig;
/**
* the entity info about the domain
*/
private Entity entity;
}

30
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailRichConfig.java
View File

@@ -1,30 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailRichConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* the entity info about the domain
*/
private EntityRichInfo entity;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfig chatDefaultConfig;
}

29
chat/core/src/main/java/com/tencent/supersonic/chat/config/Entity.java
View File

@@ -1,29 +0,0 @@
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* the entity info about the domain
*/
@Data
@AllArgsConstructor
@ToString
@NoArgsConstructor
public class Entity {
/**
* uniquely identifies an entity
*/
private Long entityId;
/**
* entity name list
*/
private List<String> names;
}

16
chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityRichInfo.java
View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfo {
/**
* entity alias
*/
private List<String> names;
private SchemaElement dimItem;
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java
View File

@@ -6,7 +6,7 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class FunctionCallConfig {
public class FunctionCallInfoConfig {
@Value("${functionCall.url:}")
private String url;
}

22
chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibility.java
View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.chat.config;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class ItemVisibility {
/**
* invisible dimensions
*/
private List<Long> blackDimIdList = new ArrayList<>();
/**
* invisible metrics
*/
private List<Long> blackMetricIdList = new ArrayList<>();
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibilityInfo.java
View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.Data;
@Data
public class ItemVisibilityInfo {
private List<Long> blackDimIdList;
private List<Long> blackMetricIdList;
private List<Long> whiteDimIdList;
private List<Long> whiteMetricIdList;
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeAdvancedConfig.java
View File

@@ -1,17 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* advanced knowledge config
*/
@Data
public class KnowledgeAdvancedConfig {
private List<String> blackList = new ArrayList<>();
private List<String> whiteList = new ArrayList<>();
private List<String> ruleList = new ArrayList<>();
}

37
chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeInfo.java
View File

@@ -1,37 +0,0 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import javax.validation.constraints.NotNull;
import lombok.Data;
/**
* information about dictionary about the domain
*/
@Data
public class KnowledgeInfo {
/**
* metricId、DimensionId、domainId
*/
private Long itemId;
private String bizName;
/**
* type: IntentionTypeEnum
* temporarily only supports dimension-related information
*/
@NotNull
private TypeEnums type = TypeEnums.DIMENSION;
private Boolean searchEnable = false;
/**
* advanced knowledge config for single item
*/
private KnowledgeAdvancedConfig knowledgeAdvancedConfig;
}

69
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java Normal file
View File

@@ -0,0 +1,69 @@
package com.tencent.supersonic.chat.mapper;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class EntityMapper implements SchemaMapper {
@Override
public void map(QueryContext queryContext) {
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
for (Long domainId : schemaMapInfo.getMatchedDomains()) {
List<SchemaElementMatch> schemaElementMatchList = schemaMapInfo.getMatchedElements(domainId);
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
continue;
}
SchemaElement entity = getEntity(domainId);
if (entity == null || entity.getId() == null) {
continue;
}
List<SchemaElementMatch> valueSchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElements) {
if (!entity.getId().equals(schemaElementMatch.getElement().getId())){
continue;
}
if (!checkExistSameEntitySchemaElements(schemaElementMatch, schemaElementMatchList)) {
SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch();
BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath);
entitySchemaElementMath.setElement(entity);
schemaElementMatchList.add(entitySchemaElementMath);
}
schemaElementMatch.getElement().setType(SchemaElementType.ID);
}
}
}
private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch,
List<SchemaElementMatch> schemaElementMatchList) {
List<SchemaElementMatch> entitySchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch ->
SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : entitySchemaElements) {
if (schemaElementMatch.getElement().getId().equals(valueSchemaElementMatch.getElement().getId())) {
return true;
}
}
return false;
}
private SchemaElement getEntity(Long domainId) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = semanticService.getDomainSchema(domainId);
if (domainSchema != null && domainSchema.getEntity() != null) {
return domainSchema.getEntity();
}
return null;
}
}

10
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java
View File

@@ -2,13 +2,9 @@ package com.tencent.supersonic.chat.mapper;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import java.util.ArrayList;
@@ -44,7 +40,7 @@ public class FuzzyNameMapper implements SchemaMapper {
}
private void detectAndAddToSchema(QueryContext queryContext, List<Term> terms, List<SchemaElement> domains,
SchemaElementType schemaElementType) {
SchemaElementType schemaElementType) {
try {
Map<String, Set<SchemaElement>> domainResultSet = getResultSet(queryContext, terms, domains);
@@ -57,7 +53,7 @@ public class FuzzyNameMapper implements SchemaMapper {
}
private Map<String, Set<SchemaElement>> getResultSet(QueryContext queryContext, List<Term> terms,
List<SchemaElement> domains) {
List<SchemaElement> domains) {
String queryText = queryContext.getRequest().getQueryText();

4
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java
View File

@@ -74,6 +74,10 @@ public class HanlpDictMapper implements SchemaMapper {
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
SchemaElement element = domainSchema.getElement(elementType, elementID);
if(Objects.isNull(element)){
log.info("element is null, elementType:{},elementID:{}", elementType, elementID);
continue;
}
if (element.getType().equals(SchemaElementType.VALUE)) {
element.setName(mapResult.getName());
}

60
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryFilterMapper.java
View File

@@ -1,27 +1,41 @@
package com.tencent.supersonic.chat.mapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
public class QueryFilterMapper implements SchemaMapper {
private Long FREQUENCY = 9999999L;
private double SIMILARITY = 1.0;
@Override
public void map(QueryContext queryContext) {
QueryRequest queryReq = queryContext.getRequest();
QueryReq queryReq = queryContext.getRequest();
Long domainId = queryReq.getDomainId();
if (domainId == null || domainId <= 0) {
return;
}
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
clearOtherSchemaElementMatch(domainId, schemaMapInfo);
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
if (schemaElementMatches == null) {
schemaElementMatches = Lists.newArrayList();
schemaMapInfo.setMatchedElements(domainId, schemaElementMatches);
}
addValueSchemaElementMatch(schemaElementMatches, queryReq.getQueryFilters());
}
private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) {
private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
if (!entry.getKey().equals(domainId)) {
entry.getValue().clear();
@@ -29,4 +43,44 @@ public class QueryFilterMapper implements SchemaMapper {
}
}
private List<SchemaElementMatch> addValueSchemaElementMatch(List<SchemaElementMatch> candidateElementMatches,
QueryFilters queryFilter) {
if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) {
return candidateElementMatches;
}
for (QueryFilter filter : queryFilter.getFilters()) {
if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) {
continue;
}
SchemaElement element = SchemaElement.builder()
.id(filter.getElementID())
.name(String.valueOf(filter.getValue()))
.type(SchemaElementType.VALUE)
.bizName(filter.getBizName())
.build();
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.element(element)
.frequency(FREQUENCY)
.word(String.valueOf(filter.getValue()))
.similarity(SIMILARITY)
.detectWord(filter.getName())
.build();
candidateElementMatches.add(schemaElementMatch);
}
return candidateElementMatches;
}
private boolean checkExistSameValueSchemaElementMatch(QueryFilter queryFilter,
List<SchemaElementMatch> schemaElementMatches) {
List<SchemaElementMatch> valueSchemaElements = schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElements) {
if (schemaElementMatch.getElement().getId().equals(queryFilter.getElementID())
&& schemaElementMatch.getWord().equals(String.valueOf(queryFilter.getValue()))) {
return true;
}
}
return false;
}
}

45
chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java
View File

@@ -3,13 +3,16 @@ package com.tencent.supersonic.chat.parser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -25,31 +28,36 @@ import java.util.stream.Collectors;
public class SatisfactionChecker {
private static final double LONG_TEXT_THRESHOLD = 0.8;
private static final double SHORT_TEXT_THRESHOLD = 0.6;
private static final double SHORT_TEXT_THRESHOLD = 0.5;
private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10;
public static final double BONUS_THRESHOLD = 100;
public static final double EMBEDDING_THRESHOLD = 0.2;
// check all the parse info in candidate
public static boolean check(QueryContext queryCtx) {
for (SemanticQuery query : queryCtx.getCandidateQueries()) {
SemanticParseInfo semanticParseInfo = query.getParseInfo();
Long domainId = semanticParseInfo.getDomainId();
List<SchemaElementMatch> schemaElementMatches = queryCtx.getMapInfo()
.getMatchedElements(domainId);
if (check(queryCtx.getRequest().getQueryText(), semanticParseInfo, schemaElementMatches)) {
return true;
if (query instanceof RuleSemanticQuery) {
if (checkRuleThreshHold(queryCtx.getRequest().getQueryText(), query.getParseInfo())) {
return true;
}
} else if (query instanceof PluginSemanticQuery) {
if (checkEmbeddingThreshold(query.getParseInfo())) {
log.info("query mode :{} satisfy check", query.getQueryMode());
return true;
}
}
}
return false;
}
private static boolean checkEmbeddingThreshold(SemanticParseInfo semanticParseInfo) {
Object object = semanticParseInfo.getProperties().get(Constants.CONTEXT);
PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(object), PluginParseResult.class);
return EMBEDDING_THRESHOLD > pluginParseResult.getDistance();
}
//check single parse info
private static boolean check(String text, SemanticParseInfo semanticParseInfo,
List<SchemaElementMatch> schemaElementMatches) {
if (semanticParseInfo.getBonus() != null && semanticParseInfo.getBonus() >= BONUS_THRESHOLD) {
return true;
}
private static boolean checkRuleThreshHold(String text, SemanticParseInfo semanticParseInfo) {
List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
if (CollectionUtils.isEmpty(schemaElementMatches)) {
return false;
}
@@ -71,6 +79,11 @@ public class SatisfactionChecker {
detectWords.add(schemaElementMatch.getDetectWord());
}
}
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
if (SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) {
detectWords.add(schemaElementMatch.getDetectWord());
}
}
for (SchemaElement schemaItem : semanticParseInfo.getMetrics()) {
detectWords.add(
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), ""));
@@ -87,7 +100,7 @@ public class SatisfactionChecker {
if (StringUtils.isNotBlank(dateText) && !dateText.equalsIgnoreCase(Constants.NULL)) {
detectWords.add(dateText);
}
detectWords.removeIf(word -> !text.contains(word));
detectWords.removeIf(word -> !text.contains(word) && !text.contains(StringUtils.reverse(word)));
//compare the length between detect words and query text
return checkThreshold(text, detectWords, semanticParseInfo);
}

165
chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingBasedParser.java
View File

@@ -5,19 +5,21 @@ import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.config.EntityRichInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.*;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -30,30 +32,43 @@ public class EmbeddingBasedParser implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
if (SatisfactionChecker.check(queryContext) || StringUtils.isBlank(embeddingConfig.getUrl())) {
if (StringUtils.isBlank(embeddingConfig.getUrl())) {
return;
}
log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext);
for (Long domainId : getDomainMatched(queryContext)) {
String text = replaceText(queryContext, domainId);
List<RecallRetrieval> embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext));
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, domainId);
if (pluginOptional.isPresent()) {
Map<String, RecallRetrieval> embeddingRetrievalMap = embeddingRetrievals.stream()
.collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1));
Plugin plugin = pluginOptional.get();
log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]",
text, domainId, plugin.getId(), plugin.getName());
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(queryContext, domainId,
plugin, embeddingRetrievalMap);
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
Set<Long> domainIds = getDomainMatched(queryContext);
String text = queryContext.getRequest().getQueryText();
if (!CollectionUtils.isEmpty(domainIds)) {
for (Long domainId : domainIds) {
List<SchemaElementMatch> schemaElementMatches = getMatchedElements(queryContext, domainId);
String textReplaced = replaceText(text, schemaElementMatches);
List<RecallRetrieval> embeddingRetrievals = recallResult(textReplaced, hasCandidateQuery(queryContext));
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, domainId);
log.info("domain id :{} embedding result, text:{} embeddingResp:{} ",domainId, textReplaced, embeddingRetrievals);
pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, domainId, textReplaced, queryContext, schemaElementMatches));
}
} else {
List<RecallRetrieval> embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext));
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, null);
pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, null, text, queryContext, Lists.newArrayList()));
}
}
private void buildQuery(Plugin plugin, List<RecallRetrieval> embeddingRetrievals,
Long domainId, String text,
QueryContext queryContext, List<SchemaElementMatch> schemaElementMatches) {
Map<String, RecallRetrieval> embeddingRetrievalMap = embeddingRetrievals.stream()
.collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1));
log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]",
text, domainId, plugin.getId(), plugin.getName());
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(domainId, plugin, text,
queryContext.getRequest(), embeddingRetrievalMap, schemaElementMatches);
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
}
private Set<Long> getDomainMatched(QueryContext queryContext) {
Long queryDomainId = queryContext.getRequest().getDomainId();
if (queryDomainId != null && queryDomainId > 0) {
@@ -62,51 +77,61 @@ public class EmbeddingBasedParser implements SemanticParser {
return queryContext.getMapInfo().getMatchedDomains();
}
private SemanticParseInfo buildSemanticParseInfo(QueryContext queryContext, Long domainId, Plugin plugin,
Map<String, RecallRetrieval> embeddingRetrievalMap) {
private SemanticParseInfo buildSemanticParseInfo(Long domainId, Plugin plugin, String text, QueryReq queryReq,
Map<String, RecallRetrieval> embeddingRetrievalMap,
List<SchemaElementMatch> schemaElementMatches) {
SchemaElement schemaElement = new SchemaElement();
schemaElement.setDomain(domainId);
schemaElement.setId(domainId);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setElementMatches(schemaElementMatches);
semanticParseInfo.setDomain(schemaElement);
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
if (Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance()) < THRESHOLD) {
semanticParseInfo.setBonus(SatisfactionChecker.BONUS_THRESHOLD);
}
double distance = Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance());
double score = text.length() * (1 - distance);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, plugin);
PluginParseResult pluginParseResult = new PluginParseResult();
pluginParseResult.setPlugin(plugin);
pluginParseResult.setRequest(queryReq);
pluginParseResult.setDistance(distance);
properties.put(Constants.CONTEXT, pluginParseResult);
semanticParseInfo.setProperties(properties);
semanticParseInfo.setElementMatches(schemaMapInfo.getMatchedElements(domainId));
fillSemanticParseInfo(queryContext, semanticParseInfo);
setEntityId(domainId, semanticParseInfo);
semanticParseInfo.setScore(score);
fillSemanticParseInfo(semanticParseInfo);
setEntity(domainId, semanticParseInfo);
return semanticParseInfo;
}
private Optional<Long> getEntityElementId(Long domainId) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRich chatConfigRich = configService.getConfigRichInfo(domainId);
EntityRichInfo entityRichInfo = chatConfigRich.getChatDetailRichConfig().getEntity();
if (entityRichInfo != null) {
SchemaElement schemaElement = entityRichInfo.getDimItem();
if (schemaElement != null) {
return Optional.of(schemaElement.getId());
}
private List<SchemaElementMatch> getMatchedElements(QueryContext queryContext, Long domainId) {
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
if (schemaElementMatches == null) {
return Lists.newArrayList();
}
return Optional.empty();
QueryReq queryReq = queryContext.getRequest();
QueryFilters queryFilters = queryReq.getQueryFilters();
if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) {
return schemaElementMatches;
}
Map<Long, Object> element = queryFilters.getFilters().stream()
.collect(Collectors.toMap(QueryFilter::getElementID, QueryFilter::getValue, (v1, v2) -> v1));
return schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType()))
.filter(schemaElementMatch ->
!element.containsKey(schemaElementMatch.getElement().getId()) || (
element.containsKey(schemaElementMatch.getElement().getId()) &&
element.get(schemaElementMatch.getElement().getId()).toString()
.equalsIgnoreCase(schemaElementMatch.getWord())
))
.collect(Collectors.toList());
}
private void setEntityId(Long domainId, SemanticParseInfo semanticParseInfo) {
Optional<Long> entityElementIdOptional = getEntityElementId(domainId);
if (entityElementIdOptional.isPresent()) {
Long entityElementId = entityElementIdOptional.get();
for (QueryFilter filter : semanticParseInfo.getDimensionFilters()) {
if (entityElementId.equals(filter.getElementID())) {
String value = String.valueOf(filter.getValue());
if (StringUtils.isNumeric(value)) {
semanticParseInfo.setEntity(Long.parseLong(value));
}
}
}
private void setEntity(Long domainId, SemanticParseInfo semanticParseInfo) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = semanticService.getDomainSchema(domainId);
if (domainSchema != null && domainSchema.getEntity() != null) {
semanticParseInfo.setEntity(domainSchema.getEntity());
}
}
@@ -120,6 +145,12 @@ public class EmbeddingBasedParser implements SemanticParser {
Map<Long, Plugin> pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p));
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId()));
if (plugin == null) {
continue;
}
if (domainId == null) {
return Optional.of(plugin);
}
if (!CollectionUtils.isEmpty(plugin.getDomainList()) && plugin.getDomainList().contains(domainId)) {
return Optional.of(plugin);
}
@@ -131,7 +162,6 @@ public class EmbeddingBasedParser implements SemanticParser {
try {
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText);
log.info("embedding result, text:{} embeddingResp:{}", embeddingText, embeddingResp);
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
if(!CollectionUtils.isEmpty(embeddingRetrievals)){
if (hasCandidateQuery) {
@@ -154,25 +184,38 @@ public class EmbeddingBasedParser implements SemanticParser {
return !CollectionUtils.isEmpty(queryContext.getCandidateQueries());
}
private void fillSemanticParseInfo(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
if (queryContext.getRequest().getQueryFilters() != null) {
semanticParseInfo.getDimensionFilters()
.addAll(queryContext.getRequest().getQueryFilters().getFilters());
private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) {
List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
if (!CollectionUtils.isEmpty(schemaElementMatches)) {
schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
}
}
protected String replaceText(QueryContext queryContext, Long domainId) {
String text = queryContext.getRequest().getQueryText();
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo().getMatchedElements(domainId);
protected String replaceText(String text, List<SchemaElementMatch> schemaElementMatches) {
if (CollectionUtils.isEmpty(schemaElementMatches)) {
return text;
}
List<SchemaElementMatch> valueSchemaElementMatches = schemaElementMatches.stream()
.filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) {
String detectWord = schemaElementMatch.getDetectWord();
if (StringUtils.isBlank(detectWord)) {
continue;
}
text = text.replace(detectWord, "");
}
return text;

16
chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingEntityResolver.java
View File

@@ -4,14 +4,10 @@ import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.parser.function.DomainResolver;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Component;
import java.util.List;
@@ -28,18 +24,6 @@ public class EmbeddingEntityResolver {
this.configService = configService;
}
public Pair<Long, Long> getDomainEntityId(QueryContext queryCtx, ChatContext chatCtx) {
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx);
ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(domainId);
SchemaElement schemaElement = chatConfigRichResp.getChatDetailRichConfig().getEntity().getDimItem();
if (schemaElement == null) {
return Pair.of(domainId, null);
}
Long entityId = getEntityValue(domainId, schemaElement.getId(), queryCtx, chatCtx);
return Pair.of(domainId, entityId);
}
private Long getEntityValue(Long domainId, Long entityElementId, QueryContext queryCtx, ChatContext chatCtx) {
Long entityId = null;

3
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/DomainResolver.java
View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.chat.parser.function;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import java.util.List;
public interface DomainResolver {
Long resolve(QueryContext queryContext, ChatContext chatCtx);
Long resolve(QueryContext queryContext, ChatContext chatCtx, List<Long> restrictiveDomains);
}

114
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionBasedParser.java
View File

@@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.FunctionCallConfig;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.chat.config.FunctionCallInfoConfig;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
@@ -22,12 +22,15 @@ import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpEntity;
@@ -44,9 +47,12 @@ public class FunctionBasedParser implements SemanticParser {
public static final double FUNCTION_BONUS_THRESHOLD = 200;
public static final double SKIP_DSL_LENGTH = 10;
@Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) {
FunctionCallConfig functionCallConfig = ContextUtils.getBean(FunctionCallConfig.class);
FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class);
PluginService pluginService = ContextUtils.getBean(PluginService.class);
String functionUrl = functionCallConfig.getUrl();
@@ -55,39 +61,57 @@ public class FunctionBasedParser implements SemanticParser {
queryCtx.getRequest().getQueryText());
return;
}
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx);
List<String> functionNames = getFunctionNames(domainId);
log.info("domainId:{},functionNames:{}", domainId, functionNames);
if (Objects.isNull(domainId) || domainId <= 0) {
Set<Long> matchedDomains = getMatchDomains(queryCtx);
List<String> functionNames = getFunctionNames(matchedDomains);
log.info("matchedDomains:{},functionNames:{}", matchedDomains, functionNames);
if (CollectionUtils.isEmpty(functionNames) || CollectionUtils.isEmpty(matchedDomains)) {
return;
}
List<PluginParseConfig> functionDOList = getFunctionDO(queryCtx.getRequest().getDomainId());
FunctionReq functionReq = FunctionReq.builder()
.queryText(queryCtx.getRequest().getQueryText())
.functionNames(functionNames).build();
.pluginConfigs(functionDOList).build();
FunctionResp functionResp = requestFunction(functionUrl, functionReq);
log.info("requestFunction result:{}", functionResp.getToolSelection());
if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) {
if (skipFunction(queryCtx, functionResp)) {
return;
}
PluginParseResult functionCallParseResult = new PluginParseResult();
String toolSelection = functionResp.getToolSelection();
Optional<Plugin> pluginOptional = pluginService.getPluginByName(toolSelection);
if (pluginOptional.isPresent()) {
toolSelection = pluginOptional.get().getType();
functionCallParseResult.setPlugin(pluginOptional.get());
if (!pluginOptional.isPresent()) {
log.info("pluginOptional is not exist:{}, skip the parse", toolSelection);
return;
}
Plugin plugin = pluginOptional.get();
toolSelection = plugin.getType();
functionCallParseResult.setPlugin(plugin);
log.info("QueryManager PluginQueryModes:{}", QueryManager.getPluginQueryModes());
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(toolSelection);
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx, plugin.getDomainList());
log.info("FunctionBasedParser domainId:{}",domainId);
if ((Objects.isNull(domainId) || domainId <= 0) && !plugin.isContainsAllDomain()) {
log.info("domain is null, skip the parse, select tool: {}", toolSelection);
return;
}
if (!plugin.getDomainList().contains(domainId) && !plugin.isContainsAllDomain()) {
return;
}
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId));
if (Objects.nonNull(domainId) && domainId > 0){
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId));
}
functionCallParseResult.setRequest(queryCtx.getRequest());
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, functionCallParseResult);
parseInfo.setProperties(properties);
parseInfo.setBonus(FUNCTION_BONUS_THRESHOLD);
parseInfo.setScore(FUNCTION_BONUS_THRESHOLD);
parseInfo.setQueryMode(semanticQuery.getQueryMode());
SchemaElement domain = new SchemaElement();
domain.setDomain(domainId);
domain.setId(domainId);
@@ -95,13 +119,61 @@ public class FunctionBasedParser implements SemanticParser {
queryCtx.getCandidateQueries().add(semanticQuery);
}
private List<String> getFunctionNames(Long domainId) {
private Set<Long> getMatchDomains(QueryContext queryCtx) {
Set<Long> result = new HashSet<>();
Long domainId = queryCtx.getRequest().getDomainId();
if (Objects.nonNull(domainId) && domainId > 0) {
result.add(domainId);
return result;
}
return queryCtx.getMapInfo().getMatchedDomains();
}
private boolean skipFunction(QueryContext queryCtx, FunctionResp functionResp) {
if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) {
return true;
}
String queryText = queryCtx.getRequest().getQueryText();
if (functionResp.getToolSelection().equalsIgnoreCase(DSLQuery.QUERY_MODE)
&& queryText.length() < SKIP_DSL_LENGTH) {
log.info("queryText length is :{}, less than the threshold :{}, skip dsl.", queryText.length(),
SKIP_DSL_LENGTH);
return true;
}
return false;
}
private List<PluginParseConfig> getFunctionDO(Long domainId) {
log.info("user decide domain:{}", domainId);
List<Plugin> plugins = PluginManager.getPlugins();
List<PluginParseConfig> functionDOList = plugins.stream().filter(o -> {
if (o.getParseModeConfig() == null) {
return false;
}
if (!CollectionUtils.isEmpty(o.getDomainList())) {//过滤掉没选主题域的插件
return true;
}
if (domainId == null || domainId <= 0L) {
return true;
} else {
return o.getDomainList().contains(domainId);
}
}).map(o -> {
PluginParseConfig functionCallConfig = JsonUtil.toObject(o.getParseModeConfig(),
PluginParseConfig.class);
return functionCallConfig;
}).collect(Collectors.toList());
log.info("getFunctionDO:{}", JsonUtil.toString(functionDOList));
return functionDOList;
}
private List<String> getFunctionNames(Set<Long> matchedDomains) {
List<Plugin> plugins = PluginManager.getPlugins();
Set<String> functionNames = plugins.stream()
.filter(entry -> ParseMode.FUNCTION_CALL.equals(entry.getParseMode()))
.filter(entry -> {
if (!CollectionUtils.isEmpty(entry.getDomainList())) {
return entry.getDomainList().contains(domainId);
if (!CollectionUtils.isEmpty(entry.getDomainList()) && !CollectionUtils.isEmpty(matchedDomains)) {
return entry.getDomainList().stream().anyMatch(matchedDomains::contains);
}
return true;
}
@@ -118,7 +190,7 @@ public class FunctionBasedParser implements SemanticParser {
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
try {
log.info("requestFunction functionReq:{}", functionReq);
log.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq));
ResponseEntity<FunctionResp> responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity,
FunctionResp.class);
log.info("requestFunction responseEntity:{},cost:{}", responseEntity,

13
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionFiled.java Normal file
View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.chat.parser.function;
import lombok.Data;
@Data
public class FunctionFiled {
private String type;
private String description;
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionReq.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.parser.function;
import java.util.List;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import lombok.Builder;
import lombok.Data;
@@ -10,6 +12,6 @@ public class FunctionReq {
private String queryText;
private List<String> functionNames;
private List<PluginParseConfig> pluginConfigs;
}

43
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/HeuristicDomainResolver.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.parser.function;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import lombok.extern.slf4j.Slf4j;
@@ -25,10 +25,10 @@ public class HeuristicDomainResolver implements DomainResolver {
Map.Entry<Long, DomainMatchResult> maxDomain = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
.sorted((o1, o2) -> {
int difference = o1.getValue().getCount() - o2.getValue().getCount();
int difference = o2.getValue().getCount() - o1.getValue().getCount();
if (difference == 0) {
return (int) ((o1.getValue().getMaxSimilarity()
- o2.getValue().getMaxSimilarity()) * 100);
return (int) ((o2.getValue().getMaxSimilarity()
- o1.getValue().getMaxSimilarity()) * 100);
}
return difference;
}).findFirst().orElse(null);
@@ -46,7 +46,7 @@ public class HeuristicDomainResolver implements DomainResolver {
* @return false will use context domain, true will use other domain , maybe include context domain
*/
protected static boolean isAllowSwitch(Map<Long, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
ChatContext chatCtx, QueryRequest searchCtx, Long domainId) {
ChatContext chatCtx, QueryReq searchCtx, Long domainId, List<Long> restrictiveDomains) {
if (!Objects.nonNull(domainId) || domainId <= 0) {
return true;
}
@@ -81,6 +81,9 @@ public class HeuristicDomainResolver implements DomainResolver {
}
}
if (CollectionUtils.isNotEmpty(restrictiveDomains) && !restrictiveDomains.contains(domainId)) {
return true;
}
// if context domain not in schemaMap , will switch
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
log.info("domainId not in schemaMap ");
@@ -118,24 +121,36 @@ public class HeuristicDomainResolver implements DomainResolver {
}
public Long resolve(QueryContext queryContext, ChatContext chatCtx) {
public Long resolve(QueryContext queryContext, ChatContext chatCtx, List<Long> restrictiveDomains) {
Long domainId = queryContext.getRequest().getDomainId();
if (Objects.nonNull(domainId) && domainId > 0) {
return domainId;
if (CollectionUtils.isNotEmpty(restrictiveDomains) && restrictiveDomains.contains(domainId)) {
return domainId;
} else {
return null;
}
}
SchemaMapInfo mapInfo = queryContext.getMapInfo();
Set<Long> matchedDomains = mapInfo.getMatchedDomains();
if (CollectionUtils.isNotEmpty(restrictiveDomains)) {
matchedDomains = matchedDomains.stream()
.filter(matchedDomain -> restrictiveDomains.contains(matchedDomain))
.collect(Collectors.toSet());
}
Map<Long, SemanticQuery> domainQueryModes = new HashMap<>();
for (Long matchedDomain : matchedDomains) {
domainQueryModes.put(matchedDomain, null);
}
if(domainQueryModes.size()==1){
return domainQueryModes.keySet().stream().findFirst().get();
}
return resolve(domainQueryModes, queryContext, chatCtx,
queryContext.getMapInfo());
queryContext.getMapInfo(),restrictiveDomains);
}
public Long resolve(Map<Long, SemanticQuery> domainQueryModes, QueryContext queryContext,
ChatContext chatCtx, SchemaMapInfo schemaMap) {
Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap);
ChatContext chatCtx, SchemaMapInfo schemaMap, List<Long> restrictiveDomains) {
Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap,restrictiveDomains);
if (selectDomain > 0) {
log.info("selectDomain {} ", selectDomain);
return selectDomain;
@@ -144,9 +159,9 @@ public class HeuristicDomainResolver implements DomainResolver {
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
}
public Long selectDomain(Map<Long, SemanticQuery> domainQueryModes, QueryRequest queryContext,
public Long selectDomain(Map<Long, SemanticQuery> domainQueryModes, QueryReq queryContext,
ChatContext chatCtx,
SchemaMapInfo schemaMap) {
SchemaMapInfo schemaMap, List<Long> restrictiveDomains) {
// if QueryContext has domainId and in domainQueryModes
if (domainQueryModes.containsKey(queryContext.getDomainId())) {
log.info("selectDomain from QueryContext [{}]", queryContext.getDomainId());
@@ -155,7 +170,7 @@ public class HeuristicDomainResolver implements DomainResolver {
// if ChatContext has domainId and in domainQueryModes
if (chatCtx.getParseInfo().getDomainId() > 0) {
Long domainId = chatCtx.getParseInfo().getDomainId();
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId)) {
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId,restrictiveDomains)) {
log.info("selectDomain from ChatContext [{}]", domainId);
return domainId;
}
@@ -163,4 +178,4 @@ public class HeuristicDomainResolver implements DomainResolver {
// default 0
return 0L;
}
}
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/Parameters.java Normal file
View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.chat.parser.function;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class Parameters {
//default: object
private String type = "object";
private Map<String, FunctionFiled> properties;
private List<String> required;
}

49
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/LLMTimeEnhancementParse.java Normal file
View File

@@ -0,0 +1,49 @@
package com.tencent.supersonic.chat.parser.llm;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.utils.ChatGptHelper;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LLMTimeEnhancementParse implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
log.info("before queryContext:{},chatContext:{}",queryContext,chatContext);
ChatGptHelper chatGptHelper = ContextUtils.getBean(ChatGptHelper.class);
String inferredTime = chatGptHelper.inferredTime(queryContext.getRequest().getQueryText());
try {
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
DateConf dateInfo = query.getParseInfo().getDateInfo();
JSONObject jsonObject = JSON.parseObject(inferredTime);
if (jsonObject.containsKey("date")){
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("date"));
dateInfo.setEndDate(jsonObject.getString("date"));
query.getParseInfo().setDateInfo(dateInfo);
}else if (jsonObject.containsKey("start")){
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("start"));
dateInfo.setEndDate(jsonObject.getString("end"));
query.getParseInfo().setDateInfo(dateInfo);
}
}
}
}catch (Exception exception){
log.error("{} parse error,this reason is:{}",LLMTimeEnhancementParse.class.getSimpleName(), (Object) exception.getStackTrace());
}
log.info("after queryContext:{},chatContext:{}",queryContext,chatContext);
}
}

20
chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java
View File

@@ -1,22 +1,28 @@
package com.tencent.supersonic.chat.parser.rule;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MAX;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MIN;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.TOPN;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.*;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.*;
@Slf4j
public class AggregateTypeParser implements SemanticParser {
@@ -29,7 +35,7 @@ public class AggregateTypeParser implements SemanticParser {
new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")),
new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")),
new AbstractMap.SimpleEntry<>(NONE, Pattern.compile("(?i)(明细)"))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,(k1,k2)->k2));
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {

56
chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java
View File

@@ -1,17 +1,25 @@
package com.tencent.supersonic.chat.parser.rule;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.util.JsonUtil;
import java.util.*;
import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
@@ -23,7 +31,8 @@ public class ContextInheritParser implements SemanticParser {
new AbstractMap.SimpleEntry<>(DIMENSION, Arrays.asList(DIMENSION, VALUE)),
new AbstractMap.SimpleEntry<>(VALUE, Arrays.asList(VALUE, DIMENSION)),
new AbstractMap.SimpleEntry<>(ENTITY, Arrays.asList(ENTITY)),
new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN))
new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN)),
new AbstractMap.SimpleEntry<>(ID, Arrays.asList(ID))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
@Override
@@ -40,7 +49,9 @@ public class ContextInheritParser implements SemanticParser {
for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) {
SchemaElementType matchType = match.getElement().getType();
// mutual exclusive element types should not be inherited
if (!containsTypes(elementMatches, MUTUAL_EXCLUSIVE_MAP.get(matchType))) {
RuleSemanticQuery ruleQuery = QueryManager.getRuleQuery(chatContext.getParseInfo().getQueryMode());
if (!containsTypes(elementMatches, matchType, ruleQuery)) {
match.setMode(SchemaElementMatch.MatchMode.INHERIT);
matchesToInherit.add(match);
}
}
@@ -53,22 +64,31 @@ public class ContextInheritParser implements SemanticParser {
}
}
private boolean containsTypes(List<SchemaElementMatch> matches, List<SchemaElementType> types) {
return matches.stream().anyMatch(m -> types.contains(m.getElement().getType()));
private boolean containsTypes(List<SchemaElementMatch> matches, SchemaElementType matchType,
RuleSemanticQuery ruleQuery) {
List<SchemaElementType> types = MUTUAL_EXCLUSIVE_MAP.get(matchType);
return matches.stream().anyMatch(m -> {
SchemaElementType type = m.getElement().getType();
if (Objects.nonNull(ruleQuery) && ruleQuery instanceof MetricSemanticQuery
&& !(ruleQuery instanceof MetricEntityQuery)) {
return types.contains(type);
}
return type.equals(matchType);
});
}
protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) {
if (queryContext.getMapInfo().getMatchedElements(
chatContext.getParseInfo().getDomainId()) == null) {
Long contextDomainId = chatContext.getParseInfo().getDomainId();
if (queryContext.getMapInfo().getMatchedElements(contextDomainId) == null) {
return false;
}
// if candidates have only one MetricDomain mode and context has value filter , count in context
if (queryContext.getCandidateQueries().size() == 1 && (queryContext.getCandidateQueries()
.get(0) instanceof MetricDomainQuery)
&& queryContext.getCandidateQueries().get(0).getParseInfo().getDomainId()
.equals(chatContext.getParseInfo().getDomainId())
&& !CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensionFilters())) {
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries().stream()
.filter(semanticQuery -> semanticQuery.getParseInfo().getDomainId().equals(contextDomainId)).collect(
Collectors.toList());
if (candidateQueries.size() == 1 && (candidateQueries.get(0) instanceof MetricDomainQuery)) {
return true;
} else {
return queryContext.getCandidateQueries().size() == 0;

1
chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java
View File

@@ -6,7 +6,6 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import java.util.*;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
@Slf4j

10
chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java
View File

@@ -46,7 +46,7 @@ public class TimeRangeParser implements SemanticParser {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
query.getParseInfo().setDateInfo(dateConf);
}
} else if(QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) {
} else if (QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) {
RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(
chatContext.getParseInfo().getQueryMode());
// inherit parse info from context
@@ -64,7 +64,7 @@ public class TimeRangeParser implements SemanticParser {
List<TimeNLP> times = TimeNLPUtil.parse(queryText);
if (times.size() > 0) {
startDate = times.get(0).getTime();
}else {
} else {
return null;
}
@@ -133,7 +133,7 @@ public class TimeRangeParser implements SemanticParser {
info.setPeriod(Constants.DAY);
}
days = days * num;
info.setDateMode(DateConf.DateMode.RECENT_UNITS);
info.setDateMode(DateConf.DateMode.RECENT);
String text = "近" + num + zhPeriod;
if (Strings.isNotEmpty(m.group("periodStr"))) {
text = m.group("periodStr");
@@ -175,11 +175,11 @@ public class TimeRangeParser implements SemanticParser {
private DateConf getDateConf(Date startDate, Date endDate) {
if (startDate == null || endDate == null) {
return null;
return null;
}
DateConf info = new DateConf();
info.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS);
info.setDateMode(DateConf.DateMode.BETWEEN);
info.setStartDate(DATE_FORMAT.format(startDate));
info.setEndDate(DATE_FORMAT.format(endDate));
return info;

42
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java
View File

@@ -53,11 +53,21 @@ public class PluginDO {
*/
private String updatedBy;
/**
*
*/
private String parseModeConfig;
/**
*
*/
private String config;
/**
*
*/
private String comment;
/**
*
* @return id
@@ -218,6 +228,22 @@ public class PluginDO {
this.updatedBy = updatedBy == null ? null : updatedBy.trim();
}
/**
*
* @return parse_mode_config
*/
public String getParseModeConfig() {
return parseModeConfig;
}
/**
*
* @param parseModeConfig
*/
public void setParseModeConfig(String parseModeConfig) {
this.parseModeConfig = parseModeConfig == null ? null : parseModeConfig.trim();
}
/**
*
* @return config
@@ -233,4 +259,20 @@ public class PluginDO {
public void setConfig(String config) {
this.config = config == null ? null : config.trim();
}
/**
*
* @return comment
*/
public String getComment() {
return comment;
}
/**
*
* @param comment
*/
public void setComment(String comment) {
this.comment = comment == null ? null : comment.trim();
}
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.persistence.mapper;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample;
import org.apache.ibatis.annotations.Mapper;

4
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.persistence.repository;
import com.tencent.supersonic.chat.config.ChatConfig;
import com.tencent.supersonic.chat.config.ChatConfigFilter;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import java.util.List;

8
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java
View File

@@ -2,17 +2,17 @@ package com.tencent.supersonic.chat.persistence.repository;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
public interface ChatQueryRepository {
PageInfo<QueryResponse> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId);
PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId);
void createChatQuery(QueryResult queryResult, QueryRequest queryContext, ChatContext chatCtx);
void createChatQuery(QueryResult queryResult, ChatContext chatCtx);
ChatQueryDO getLastChatQuery(long chatId);

4
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatConfigRepositoryImpl.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.persistence.repository.impl;
import com.tencent.supersonic.chat.config.ChatConfig;
import com.tencent.supersonic.chat.config.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.config.ChatConfigFilterInternal;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO;
import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository;
import com.tencent.supersonic.chat.utils.ChatConfigHelper;

22
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java
View File

@@ -3,12 +3,12 @@ package com.tencent.supersonic.chat.persistence.repository.impl;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample.Criteria;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.persistence.mapper.ChatQueryDOMapper;
import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
@@ -35,7 +35,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
}
@Override
public PageInfo<QueryResponse> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) {
public PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) {
ChatQueryDOExample example = new ChatQueryDOExample();
example.setOrderByClause("question_id desc");
Criteria criteria = example.createCriteria();
@@ -46,7 +46,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
pageQueryInfoCommend.getPageSize())
.doSelectPageInfo(() -> chatQueryDOMapper.selectByExampleWithBLOBs(example));
PageInfo<QueryResponse> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo);
PageInfo<QueryResp> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo);
chatQueryVOPageInfo.setList(
pageInfo.getList().stream().map(this::convertTo)
.sorted(Comparator.comparingInt(o -> o.getQuestionId().intValue()))
@@ -54,8 +54,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatQueryVOPageInfo;
}
private QueryResponse convertTo(ChatQueryDO chatQueryDO) {
QueryResponse queryResponse = new QueryResponse();
private QueryResp convertTo(ChatQueryDO chatQueryDO) {
QueryResp queryResponse = new QueryResp();
BeanUtils.copyProperties(chatQueryDO, queryResponse);
QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class);
queryResult.setQueryId(chatQueryDO.getQuestionId());
@@ -64,16 +64,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
}
@Override
public void createChatQuery(QueryResult queryResult, QueryRequest queryRequest, ChatContext chatCtx) {
public void createChatQuery(QueryResult queryResult, ChatContext chatCtx) {
ChatQueryDO chatQueryDO = new ChatQueryDO();
chatQueryDO.setChatId(Long.valueOf(queryRequest.getChatId()));
chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId()));
chatQueryDO.setCreateTime(new java.util.Date());
chatQueryDO.setUserName(queryRequest.getUser().getName());
chatQueryDO.setUserName(chatCtx.getUser());
chatQueryDO.setQueryState(queryResult.getQueryState().ordinal());
chatQueryDO.setQueryText(queryRequest.getQueryText());
chatQueryDO.setQueryText(chatCtx.getQueryText());
chatQueryDO.setQueryResult(JsonUtil.toString(queryResult));
chatQueryDOMapper.insert(chatQueryDO);
ChatQueryDO lastChatQuery = getLastChatQuery(queryRequest.getChatId());
ChatQueryDO lastChatQuery = getLastChatQuery(chatCtx.getChatId());
Long queryId = lastChatQuery.getQuestionId();
queryResult.setQueryId(queryId);
}

41
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java
View File

@@ -1,37 +1,58 @@
package com.tencent.supersonic.chat.plugin;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Data
public class Plugin extends RecordInfo {
private Long id;
//plugin type WEB_PAGE WEB_SERVICE
/***
* plugin type WEB_PAGE WEB_SERVICE
*/
private String type;
private List<Long> domainList;
private List<Long> domainList = Lists.newArrayList();
//description, for parsing
/**
* description, for parsing
*/
private String pattern;
//parse
/**
* parse
*/
private ParseMode parseMode;
private String parseModeConfig;
private String name;
//config for different plugin type
/**
* config for different plugin type
*/
private String config;
public List<String> getPatterns() {
return Stream.of(getPattern().split("\\|")).collect(Collectors.toList());
private String comment;
public List<String> getExampleQuestionList() {
if (StringUtils.isNotBlank(parseModeConfig)) {
PluginParseConfig pluginParseConfig = JSONObject.parseObject(parseModeConfig, PluginParseConfig.class);
return pluginParseConfig.getExamples();
}
return Lists.newArrayList();
}
public boolean isContainsAllDomain() {
return CollectionUtils.isNotEmpty(domainList) && domainList.contains(-1L);
}
}

51
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java
View File

@@ -16,6 +16,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.context.event.EventListener;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
@@ -40,7 +41,8 @@ public class PluginManager {
public static List<Plugin> getPlugins() {
PluginService pluginService = ContextUtils.getBean(PluginService.class);
List<Plugin> pluginList = pluginService.getPluginList();
List<Plugin> pluginList = pluginService.getPluginList().stream().filter(plugin ->
CollectionUtils.isNotEmpty(plugin.getDomainList())).collect(Collectors.toList());
pluginList.addAll(internalPluginMap.values());
return new ArrayList<>(pluginList);
}
@@ -48,7 +50,7 @@ public class PluginManager {
@EventListener
public void addPlugin(PluginAddEvent pluginAddEvent) {
Plugin plugin = pluginAddEvent.getPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) {
if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) {
requestEmbeddingPluginAdd(convert(Lists.newArrayList(plugin)));
}
}
@@ -57,10 +59,10 @@ public class PluginManager {
public void updatePlugin(PluginUpdateEvent pluginUpdateEvent) {
Plugin oldPlugin = pluginUpdateEvent.getOldPlugin();
Plugin newPlugin = pluginUpdateEvent.getNewPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(oldPlugin.getParseMode())) {
if (CollectionUtils.isNotEmpty(oldPlugin.getExampleQuestionList())) {
requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(oldPlugin)));
}
if (ParseMode.EMBEDDING_RECALL.equals(newPlugin.getParseMode())) {
if (CollectionUtils.isNotEmpty(newPlugin.getExampleQuestionList())) {
requestEmbeddingPluginAdd(convert(Lists.newArrayList(newPlugin)));
}
}
@@ -68,29 +70,30 @@ public class PluginManager {
@EventListener
public void delPlugin(PluginAddEvent pluginAddEvent) {
Plugin plugin = pluginAddEvent.getPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) {
if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) {
requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(plugin)));
}
}
public void requestEmbeddingPluginDelete(Set<String> ids) {
if(CollectionUtils.isEmpty(ids)){
if (CollectionUtils.isEmpty(ids)) {
return;
}
doRequest(embeddingConfig.getDeletePath(), JSONObject.toJSONString(ids));
}
public void requestEmbeddingPluginAdd(List<Map<String,String>> maps) {
if(CollectionUtils.isEmpty(maps)){
public void requestEmbeddingPluginAdd(List<Map<String, String>> maps) {
if (CollectionUtils.isEmpty(maps)) {
return;
}
doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps));
doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps));
}
public void doRequest(String path, String jsonBody) {
String url = embeddingConfig.getUrl()+ path;
if (Strings.isEmpty(embeddingConfig.getUrl())) {
return;
}
String url = embeddingConfig.getUrl() + path;
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url));
@@ -99,7 +102,8 @@ public class PluginManager {
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] equest body :{}, url:{}", jsonBody, url);
ResponseEntity<String> responseEntity =
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<String>() {});
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<String>() {
});
log.info("[embedding] result body:{}", responseEntity);
}
@@ -111,7 +115,7 @@ public class PluginManager {
}
public EmbeddingResp recognize(String embeddingText) {
String url = embeddingConfig.getUrl()+ embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult();
String url = embeddingConfig.getUrl() + embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url));
@@ -121,10 +125,11 @@ public class PluginManager {
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<List<EmbeddingResp>>() {});
log.info("[embedding] recognize result body:{}",embeddingResponseEntity);
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<List<EmbeddingResp>>() {
});
log.info("[embedding] recognize result body:{}", embeddingResponseEntity);
List<EmbeddingResp> embeddingResps = embeddingResponseEntity.getBody();
if(CollectionUtils.isNotEmpty(embeddingResps)){
if (CollectionUtils.isNotEmpty(embeddingResps)) {
for (EmbeddingResp embeddingResp : embeddingResps) {
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
@@ -136,13 +141,13 @@ public class PluginManager {
throw new RuntimeException("get embedding result failed");
}
public List<Map<String, String>> convert(List<Plugin> plugins){
public List<Map<String, String>> convert(List<Plugin> plugins) {
List<Map<String, String>> maps = Lists.newArrayList();
for(Plugin plugin : plugins){
List<String> patterns = plugin.getPatterns();
for (Plugin plugin : plugins) {
List<String> exampleQuestions = plugin.getExampleQuestionList();
int num = 0;
for(String pattern : patterns){
Map<String,String> map = new HashMap<>();
for (String pattern : exampleQuestions) {
Map<String, String> map = new HashMap<>();
map.put("preset_query_id", generateUniqueEmbeddingId(num, plugin.getId()));
map.put("preset_query", pattern);
maps.add(map);
@@ -155,7 +160,7 @@ public class PluginManager {
private Set<String> getEmbeddingId(List<Plugin> plugins) {
Set<String> embeddingIdSet = new HashSet<>();
for (Map<String, String> map : convert(plugins)) {
embeddingIdSet.addAll(map.keySet());
embeddingIdSet.add(map.get("preset_query_id"));
}
return embeddingIdSet;
}

21
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseConfig.java Normal file
View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.chat.parser.function.Parameters;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
@Data
public class PluginParseConfig implements Serializable {
private String name;
private String description;
public Parameters parameters;
public List<String> examples;
}

5
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseResult.java
View File

@@ -1,11 +1,12 @@
package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.Data;
@Data
public class PluginParseResult {
private Plugin plugin;
private QueryRequest request;
private QueryReq request;
private double distance;
}

69
chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java
View File

@@ -5,39 +5,46 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class HeuristicQuerySelector implements QuerySelector {
@Override
public SemanticQuery select(List<SemanticQuery> candidateQueries) {
double maxScore = 0;
SemanticQuery pickedQuery = null;
if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) {
return candidateQueries.get(0);
}
for (SemanticQuery query : candidateQueries) {
SemanticParseInfo semanticParse = query.getParseInfo();
double score = computeScore(semanticParse);
if (score > maxScore) {
maxScore = score;
pickedQuery = query;
}
log.info("candidate query (domain={}, queryMode={}) with score={}",
semanticParse.getDomainName(), semanticParse.getQueryMode(), score);
}
private static final double MATCH_INHERIT_PENALTY = 0.5;
private static final double MATCH_CURRENT_REWORD = 2;
private static final double CANDIDATE_THRESHOLD = 0.2;
return pickedQuery;
@Override
public List<SemanticQuery> select(List<SemanticQuery> candidateQueries) {
List<SemanticQuery> selectedQueries = new ArrayList<>();
if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) {
selectedQueries.addAll(candidateQueries);
} else {
OptionalDouble maxScoreOp = candidateQueries.stream().mapToDouble(
q -> computeScore(q.getParseInfo())).max();
if (maxScoreOp.isPresent()) {
double maxScore = maxScoreOp.getAsDouble();
candidateQueries.stream().forEach(query -> {
SemanticParseInfo semanticParse = query.getParseInfo();
if ((maxScore - semanticParse.getScore()) / maxScore <= CANDIDATE_THRESHOLD) {
selectedQueries.add(query);
}
log.info("candidate query (domain={}, queryMode={}) with score={}",
semanticParse.getDomainName(), semanticParse.getQueryMode(), semanticParse.getScore());
});
}
}
return selectedQueries;
}
private double computeScore(SemanticParseInfo semanticParse) {
double score = 0;
double totalScore = 0;
Map<SchemaElementType, SchemaElementMatch> maxSimilarityMatch = new HashMap<>();
for (SchemaElementMatch match : semanticParse.getElementMatches()) {
@@ -49,13 +56,19 @@ public class HeuristicQuerySelector implements QuerySelector {
}
for (SchemaElementMatch match : maxSimilarityMatch.values()) {
score +=
Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity();
double matchScore = Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity();
if (match.equals(SchemaElementMatch.MatchMode.INHERIT)) {
matchScore *= MATCH_INHERIT_PENALTY;
} else {
matchScore *= MATCH_CURRENT_REWORD;
}
totalScore += matchScore;
}
// bonus is a special construct to control the final score
score += semanticParse.getBonus();
// original score in parse info acts like an extra bonus
totalScore += semanticParse.getScore();
semanticParse.setScore(totalScore);
return score;
return totalScore;
}
}

33
chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java
View File

@@ -3,6 +3,9 @@ package com.tencent.supersonic.chat.query;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -22,6 +25,14 @@ public class QueryManager {
}
}
public static SemanticQuery createQuery(String queryMode) {
if (containsRuleQuery(queryMode)) {
return createRuleQuery(queryMode);
} else {
return createPluginQuery(queryMode);
}
}
public static RuleSemanticQuery createRuleQuery(String queryMode) {
RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode);
if (Objects.isNull(semanticQuery)) {
@@ -45,7 +56,6 @@ public class QueryManager {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
}
public static boolean containsRuleQuery(String queryMode) {
if (queryMode == null) {
return false;
@@ -53,6 +63,27 @@ public class QueryManager {
return ruleQueryMap.containsKey(queryMode);
}
public static boolean isMetricQuery(String queryMode) {
if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) {
return false;
}
return ruleQueryMap.get(queryMode) instanceof MetricSemanticQuery;
}
public static boolean isEntityQuery(String queryMode) {
if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) {
return false;
}
return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery;
}
public static RuleSemanticQuery getRuleQuery(String queryMode) {
if (queryMode == null) {
return null;
}
return ruleQueryMap.get(queryMode);
}
public static List<RuleSemanticQuery> getRuleQueries() {
return new ArrayList<>(ruleQueryMap.values());
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/QuerySelector.java
View File

@@ -9,5 +9,5 @@ import java.util.List;
**/
public interface QuerySelector {
SemanticQuery select(List<SemanticQuery> candidateQueries);
List<SemanticQuery> select(List<SemanticQuery> candidateQueries);
}

37
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/ParamOption.java Normal file
View File

@@ -0,0 +1,37 @@
package com.tencent.supersonic.chat.query.plugin;
import lombok.Data;
@Data
public class ParamOption {
private ParamType paramType;
private OptionType optionType;
private String key;
private String name;
private String keyAlias;
private Long domainId;
private Long elementId;
private Object value;
/**
* CUSTOM: the value is specified by the user
* SEMANTIC: the value of element
* FORWARD: only forward
*/
public enum ParamType {
CUSTOM, SEMANTIC, FORWARD
}
public enum OptionType {
REQUIRED, OPTIONAL
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java
View File

@@ -1,22 +1,14 @@
package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
@Data
public class WebBase {
private String url;
//key, id of schema element
private Map<String, Object> params = new HashMap<>();
//key, value of shcema element
private Map<String, Object> valueParams = new HashMap<>();
//only forward
private Map<String, Object> forwardParam = new HashMap<>();
private List<ParamOption> paramOptions = Lists.newArrayList();
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBaseResult.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class WebBaseResult {
private String url;
private List<ParamOption> params = Lists.newArrayList();
}

3
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/dsl/DSLQuery.java
View File

@@ -62,8 +62,7 @@ public class DSLQuery extends PluginSemanticQuery {
@Override
public QueryResult execute(User user) {
PluginParseResult functionCallParseResult = (PluginParseResult) parseInfo.getProperties()
.get(Constants.CONTEXT);
PluginParseResult functionCallParseResult =JsonUtil.toObject(JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)),PluginParseResult.class);
Long domainId = parseInfo.getDomainId();
LLMResp llmResp = requestLLM(functionCallParseResult, domainId);
if (Objects.isNull(llmResp)) {

71
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java
View File

@@ -1,15 +1,19 @@
package com.tencent.supersonic.chat.query.plugin.webpage;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.ParamOption;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants;
@@ -18,10 +22,9 @@ import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Component
@@ -44,13 +47,14 @@ public class WebPageQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
Plugin plugin = (Plugin) properties.get(Constants.CONTEXT);
WebPageResponse webPageResponse = buildResponse(plugin);
PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
WebPageResponse webPageResponse = buildResponse(pluginParseResult.getPlugin());
queryResult.setResponse(webPageResponse);
if (parseInfo.getDomainId() != null && parseInfo.getDomainId() > 0
&& parseInfo.getEntity() != null && parseInfo.getEntity() > 0) {
ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId());
updateSemanticParse(chatConfigRichResp, parseInfo.getEntity());
&& parseInfo.getEntity() != null && Objects.nonNull(parseInfo.getEntity().getId())
&& parseInfo.getEntity().getId() > 0) {
ChatConfigRichResp chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId());
updateSemanticParse(chatConfigRichResp);
EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class).getEntityInfo(parseInfo, user);
queryResult.setEntityInfo(entityInfo);
} else {
@@ -60,8 +64,7 @@ public class WebPageQuery extends PluginSemanticQuery {
return queryResult;
}
private void updateSemanticParse(ChatConfigRich chatConfigRichResp, Long entityId) {
parseInfo.setEntity(entityId);
private void updateSemanticParse(ChatConfigRichResp chatConfigRichResp) {
SchemaElement domain = new SchemaElement();
domain.setId(chatConfigRichResp.getDomainId());
domain.setName(chatConfigRichResp.getDomainName());
@@ -74,35 +77,43 @@ public class WebPageQuery extends PluginSemanticQuery {
webPageResponse.setPluginId(plugin.getId());
webPageResponse.setPluginType(plugin.getType());
WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class);
fillWebPage(webPage);
webPageResponse.setWebPage(webPage);
WebBaseResult webBaseResult = buildWebPageResult(webPage);
webPageResponse.setWebPage(webBaseResult);
return webPageResponse;
}
private void fillWebPage(WebBase webPage) {
List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches();
private WebBaseResult buildWebPageResult(WebBase webPage) {
WebBaseResult webBaseResult = new WebBaseResult();
webBaseResult.setUrl(webPage.getUrl());
Map<String, Object> elementValueMap = getElementMap();
if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) {
for (ParamOption paramOption : webPage.getParamOptions()) {
if (!ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType())) {
continue;
}
String elementId = String.valueOf(paramOption.getElementId());
Object elementValue = elementValueMap.get(elementId);
paramOption.setValue(elementValue);
}
}
webBaseResult.setParams(webPage.getParamOptions());
return webBaseResult;
}
private Map<String, Object> getElementMap() {
Map<String, Object> elementValueMap = new HashMap<>();
if (!CollectionUtils.isEmpty(schemaElementMatchList) && !CollectionUtils.isEmpty(webPage.getParams()) ) {
List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches();
if (!CollectionUtils.isEmpty(schemaElementMatchList)) {
schemaElementMatchList.stream()
.filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.sorted(Comparator.comparingDouble(SchemaElementMatch::getSimilarity))
.forEach(schemaElementMatch ->
elementValueMap.put(String.valueOf(schemaElementMatch.getElement().getId()),
schemaElementMatch.getWord()));
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())) {
parseInfo.getDimensionFilters().forEach(
filter -> elementValueMap.put(String.valueOf(filter.getElementID()), filter.getValue())
);
}
Map<String, Object> params = webPage.getParams();
for (Map.Entry<String, Object> entry : params.entrySet()) {
String key = entry.getKey();
String elementId = String.valueOf(entry.getValue());
Object elementValue = elementValueMap.get(elementId);
webPage.getValueParams().put(key, elementValue);
}
return elementValueMap;
}
}

5
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.query.plugin.webpage;
import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
import lombok.Data;
import java.util.List;
@@ -15,8 +16,8 @@ public class WebPageResponse {
private String description;
private WebBase webPage;
private WebBaseResult webPage;
private List<WebBase> moreWebPage;
private List<WebBaseResult> moreWebPage;
}

46
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java
View File

@@ -1,22 +1,28 @@
package com.tencent.supersonic.chat.query.plugin.webservice;
import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.ParamOption;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.*;
import java.net.URI;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.springframework.http.*;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.util.HashMap;
import java.util.Map;
@@ -27,7 +33,7 @@ public class WebServiceQuery extends PluginSemanticQuery {
public static String QUERY_MODE = "WEB_SERVICE";
private S2ThreadContext s2ThreadContext;
private RestTemplate restTemplate;
public WebServiceQuery() {
QueryManager.register(this);
@@ -43,13 +49,9 @@ public class WebServiceQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
PluginParseResult pluginParseResult = (PluginParseResult) properties.get(Constants.CONTEXT);
PluginParseResult pluginParseResult =JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)),PluginParseResult.class);
WebServiceResponse webServiceResponse = buildResponse(pluginParseResult);
Object object = webServiceResponse.getResult();
Map<String,Object> data=JsonUtil.toMap(JsonUtil.toString(object),String.class,Object.class);
queryResult.setQueryResults((List<Map<String, Object>>) data.get("resultList"));
queryResult.setQueryColumns((List<QueryColumn>) data.get("columns"));
//queryResult.setResponse(webServiceResponse);
queryResult.setResponse(webServiceResponse);
queryResult.setQueryState(QueryState.SUCCESS);
parseInfo.setProperties(null);
return queryResult;
@@ -60,22 +62,22 @@ public class WebServiceQuery extends PluginSemanticQuery {
Plugin plugin = pluginParseResult.getPlugin();
WebBase webBase = JsonUtil.toObject(plugin.getConfig(), WebBase.class);
webServiceResponse.setWebBase(webBase);
//http todo
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
String authHeader = s2ThreadContext.get().getToken();
log.info("authHeader:{}", authHeader);
List<ParamOption> paramOptions = webBase.getParamOptions();
Map<String, Object> params = new HashMap<>();
paramOptions.forEach(o -> params.put(o.getKey(), o.getValue()));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<String> entity = new HttpEntity<>(JSON.toJSONString(params), headers);
URI requestUrl = UriComponentsBuilder.fromHttpUrl(webBase.getUrl()).build().encode().toUri();
ResponseEntity responseEntity = null;
Object objectResponse = null;
restTemplate = ContextUtils.getBean(RestTemplate.class);
try {
Map<String, String> headers = new HashMap<>();
headers.put("Authorization", authHeader);
Map<String, String> params = new HashMap<>();
params.put("queryText", pluginParseResult.getRequest().getQueryText());
HttpClientResult httpClientResult = HttpClientUtils.doGet(webBase.getUrl(), headers, params);
log.info(" response body:{}", httpClientResult.getContent());
Map<String, Object> result = JsonUtil.toMap(JsonUtil.toString(httpClientResult.getContent()), String.class, Object.class);
log.info(" result:{}", result);
Map<String, Object> data = JsonUtil.toMap(JsonUtil.toString(result.get("data")), String.class, Object.class);
log.info(" data:{}", data);
webServiceResponse.setResult(data);
responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class);
objectResponse = responseEntity.getBody();
log.info("objectResponse:{}", objectResponse);
Map<String, Object> response = JsonUtil.objectToMap(objectResponse);
webServiceResponse.setResult(response);
} catch (Exception e) {
log.info("Exception:{}", e.getMessage());
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/QueryMatcher.java
View File

@@ -26,8 +26,6 @@ public class QueryMatcher {
private boolean supportOrderBy;
private List<AggregateTypeEnum> orderByTypes = Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN,
AggregateTypeEnum.TOPN);
private Long FREQUENCY = 9999999L;
private double SIMILARITY = 1.0;
public QueryMatcher() {
for (SchemaElementType type : SchemaElementType.values()) {
@@ -52,11 +50,10 @@ public class QueryMatcher {
* @return a list of all matched schema elements,
* empty list if no matches can be found
*/
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches, QueryFilters queryFilters) {
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches) {
List<SchemaElementMatch> elementMatches = new ArrayList<>();
List<SchemaElementMatch> schemaElementMatchWithQueryFilter = addSchemaElementMatch(candidateElementMatches, queryFilters);
HashMap<SchemaElementType, Integer> schemaElementTypeCount = new HashMap<>();
for (SchemaElementMatch schemaElementMatch : schemaElementMatchWithQueryFilter) {
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
SchemaElementType schemaElementType = schemaElementMatch.getElement().getType();
if (schemaElementTypeCount.containsKey(schemaElementType)) {
schemaElementTypeCount.put(schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1);
@@ -75,7 +72,7 @@ public class QueryMatcher {
}
// add element match if its element type is not declared as unused
for (SchemaElementMatch elementMatch : schemaElementMatchWithQueryFilter) {
for (SchemaElementMatch elementMatch : candidateElementMatches) {
QueryMatchOption elementOption = elementOptionMap.get(elementMatch.getElement().getType());
if (Objects.nonNull(elementOption) && !elementOption.getSchemaElementOption()
.equals(QueryMatchOption.OptionType.UNUSED)) {
@@ -86,32 +83,6 @@ public class QueryMatcher {
return elementMatches;
}
private List<SchemaElementMatch> addSchemaElementMatch(List<SchemaElementMatch> candidateElementMatches, QueryFilters queryFilter) {
List<SchemaElementMatch> schemaElementMatchWithQueryFilter = new ArrayList<>(candidateElementMatches);
if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) {
return schemaElementMatchWithQueryFilter;
}
QueryMatchOption queryMatchOption = elementOptionMap.get(SchemaElementType.VALUE);
if (queryMatchOption != null && QueryMatchOption.OptionType.REQUIRED.equals(queryMatchOption.getSchemaElementOption())) {
for (QueryFilter filter : queryFilter.getFilters()) {
SchemaElement element = SchemaElement.builder()
.id(filter.getElementID())
.name(String.valueOf(filter.getValue()))
.type(SchemaElementType.VALUE)
.build();
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.element(element)
.frequency(FREQUENCY)
.word(String.valueOf(filter.getValue()))
.similarity(SIMILARITY)
.detectWord(Constants.EMPTY)
.build();
schemaElementMatchWithQueryFilter.add(schemaElementMatch);
}
}
return schemaElementMatchWithQueryFilter;
}
private int getCount(HashMap<SchemaElementType, Integer> schemaElementTypeCount,
SchemaElementType schemaElementType) {
if (schemaElementTypeCount.containsKey(schemaElementType)) {

83
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -9,9 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
@@ -24,6 +22,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import java.io.Serializable;
import java.util.*;
@@ -42,33 +41,51 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
return queryMatcher.match(candidateElementMatches, queryCtx.getRequest().getQueryFilters());
return queryMatcher.match(candidateElementMatches);
}
public void fillParseInfo(Long domainId, ChatContext chatContext){
public void fillParseInfo(Long domainId, ChatContext chatContext) {
parseInfo.setQueryMode(getQueryMode());
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRich chatConfig = configService.getConfigRichInfo(domainId);
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = schemaService.getDomainSchema(domainId);
fillSchemaElement(parseInfo, domainSchema, chatConfig);
fillSchemaElement(parseInfo, domainSchema);
// inherit date info from context
if (parseInfo.getDateInfo() == null && chatContext.getParseInfo().getDateInfo() != null) {
if (parseInfo.getDateInfo() == null && chatContext.getParseInfo().getDateInfo() != null
&& isSameQueryMode(getQueryMode(), chatContext.getParseInfo().getQueryMode())) {
log.info("inherit date info from context");
parseInfo.setDateInfo(chatContext.getParseInfo().getDateInfo());
}
}
private void fillSchemaElement(SemanticParseInfo parseInfo, DomainSchema domainSchema, ChatConfigRich chaConfigRich) {
public boolean isSameQueryMode(String queryModeQuery, String queryModeChat) {
if (Strings.isNotEmpty(queryModeQuery) && Strings.isNotEmpty(queryModeChat)) {
return QueryManager.isEntityQuery(queryModeQuery) && QueryManager.isEntityQuery(queryModeChat)
|| QueryManager.isMetricQuery(queryModeQuery) && QueryManager.isMetricQuery(queryModeChat);
}
return true;
}
private void fillSchemaElement(SemanticParseInfo parseInfo, DomainSchema domainSchema) {
parseInfo.setDomain(domainSchema.getDomain());
Map<Long, List<SchemaElementMatch>> dim2Values = new HashMap<>();
Map<Long, List<SchemaElementMatch>> id2Values = new HashMap<>();
for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
SchemaElement element = schemaMatch.getElement();
switch (element.getType()) {
case ID:
SchemaElement entityElement = domainSchema.getElement(SchemaElementType.ENTITY, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
break;
case VALUE:
SchemaElement dimElement = domainSchema.getElement(SchemaElementType.DIMENSION, element.getId());
if (dimElement != null) {
@@ -85,10 +102,41 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
case METRIC:
parseInfo.getMetrics().add(element);
break;
case ENTITY:
parseInfo.setEntity(element);
break;
default:
}
}
if (!id2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
SchemaElement entity = domainSchema.getElement(SchemaElementType.ENTITY, entry.getKey());
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(schemaMatch.getWord());
dimensionFilter.setBizName(entity.getBizName());
dimensionFilter.setName(entity.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.getDimensionFilters().add(dimensionFilter);
parseInfo.setEntity(domainSchema.getEntity());
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
dimensionFilter.setValue(vals);
dimensionFilter.setBizName(entity.getBizName());
dimensionFilter.setName(entity.getName());
dimensionFilter.setOperator(FilterOperatorEnum.IN);
dimensionFilter.setElementID(entry.getKey());
parseInfo.getDimensionFilters().add(dimensionFilter);
}
}
}
if (!dim2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dim2Values.entrySet()) {
SchemaElement dimension = domainSchema.getElement(SchemaElementType.DIMENSION, entry.getKey());
@@ -102,7 +150,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.getDimensionFilters().add(dimensionFilter);
setEntityId(schemaMatch.getWord(), chaConfigRich, parseInfo);
parseInfo.setEntity(domainSchema.getEntity());
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
@@ -118,16 +166,6 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
}
}
public void setEntityId(String value, ChatConfigRich chaConfigRichDesc,
SemanticParseInfo semanticParseInfo) {
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
SchemaElement dimSchemaResp = chaConfigRichDesc.getChatDetailRichConfig().getEntity().getDimItem();
if (Objects.nonNull(dimSchemaResp) && StringUtils.isNumeric(value)) {
semanticParseInfo.setEntity(Long.valueOf(value));
}
}
}
@Override
public QueryResult execute(User user) {
@@ -202,12 +240,13 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
return parseInfo;
}
@Override
public void setParseInfo(SemanticParseInfo parseInfo) {
this.parseInfo = parseInfo;
}
public static List<RuleSemanticQuery> resolve(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryContext) {
QueryContext queryContext) {
List<RuleSemanticQuery> matchedQueries = new ArrayList<>();
for (RuleSemanticQuery semanticQuery : QueryManager.getRuleQueries()) {
List<SchemaElementMatch> matches = semanticQuery.match(candidateElementMatches, queryContext);

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityDetailQuery.java
View File

@@ -14,7 +14,7 @@ public class EntityDetailQuery extends EntitySemanticQuery {
public EntityDetailQuery() {
super();
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
.addOption(ID, REQUIRED, AT_LEAST, 1);
}
@Override

5
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityFilterQuery.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.query.rule.entity;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -16,7 +16,8 @@ public class EntityFilterQuery extends EntityListQuery {
public EntityFilterQuery() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
queryMatcher.addOption(VALUE, OPTIONAL, AT_LEAST, 0);
queryMatcher.addOption(ID, OPTIONAL, AT_LEAST, 0);
}
@Override

24
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityListQuery.java
View File

@@ -1,37 +1,43 @@
package com.tencent.supersonic.chat.query.rule.entity;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
public abstract class EntityListQuery extends EntitySemanticQuery{
public abstract class EntityListQuery extends EntitySemanticQuery {
@Override
public void fillParseInfo(Long domainId, ChatContext chatContext){
public void fillParseInfo(Long domainId, ChatContext chatContext) {
super.fillParseInfo(domainId, chatContext);
this.addEntityDetailAndOrderByMetric(parseInfo);
}
private void addEntityDetailAndOrderByMetric(SemanticParseInfo parseInfo) {
if (parseInfo.getDomainId() > 0L) {
Long domainId = parseInfo.getDomainId();
if (Objects.nonNull(domainId) && domainId > 0L) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo(
parseInfo.getDomainId());
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(parseInfo.getDomainId());
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = schemaService.getDomainSchema(domainId);
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
&& Objects.nonNull(domainSchema) && Objects.nonNull(domainSchema.getEntity())) {
Set<SchemaElement> dimensions = new LinkedHashSet();
Set<SchemaElement> metrics = new LinkedHashSet();
Set<Order> orders = new LinkedHashSet();
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
if (chatDefaultConfig != null) {
chatDefaultConfig.getMetrics().stream()
.forEach(metric -> {

12
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java
View File

@@ -4,9 +4,9 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.common.pojo.DateConf;
@@ -85,8 +85,8 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery {
parseInfo.setLimit(ENTITY_MAX_RESULTS);
if (parseInfo.getDateInfo() == null) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRich chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId());
ChatDefaultRichConfig defaultConfig = chatConfig.getChatDetailRichConfig().getChatDefaultConfig();
ChatConfigRichResp chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId());
ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatDetailRichConfig().getChatDefaultConfig();
int unit = 1;
if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) {
@@ -94,7 +94,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery {
}
String date = LocalDate.now().plusDays(-unit).toString();
DateConf dateInfo = new DateConf();
dateInfo.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS);
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);

25
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityTopNQuery.java
View File

@@ -1,25 +0,0 @@
package com.tencent.supersonic.chat.query.rule.entity;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import org.springframework.stereotype.Component;
@Component
public class EntityTopNQuery extends EntityListQuery {
public static final String QUERY_MODE = "ENTITY_LIST_TOPN";
public EntityTopNQuery() {
super();
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
.setSupportOrderBy(true);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
}

26
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricDomainQuery.java
View File

@@ -1,17 +1,12 @@
package com.tencent.supersonic.chat.query.rule.metric;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import java.util.Objects;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DOMAIN;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_MOST;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import org.springframework.stereotype.Component;
@Component
public class MetricDomainQuery extends MetricSemanticQuery {
@@ -31,14 +26,7 @@ public class MetricDomainQuery extends MetricSemanticQuery {
@Override
public QueryResult execute(User user) {
QueryResult queryResult = super.execute(user);
if (!Objects.isNull(queryResult)) {
QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp();
queryResp.setColumns(queryResult.getQueryColumns());
queryResp.setResultList(queryResult.getQueryResults());
AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class)
.getAggregateInfo(user, parseInfo, queryResp);
queryResult.setAggregateInfo(aggregateInfo);
}
fillAggregateInfo(user, queryResult);
return queryResult;
}

92
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java Normal file
View File

@@ -0,0 +1,92 @@
package com.tencent.supersonic.chat.query.rule.metric;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
@Slf4j
@Component
public class MetricEntityQuery extends MetricSemanticQuery {
public static final String QUERY_MODE = "METRIC_ENTITY";
public MetricEntityQuery() {
super();
queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1)
.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public QueryResult execute(User user) {
if (!isMultiStructQuery()) {
QueryResult queryResult = super.execute(user);
fillAggregateInfo(user, queryResult);
return queryResult;
}
return super.multiStructExecute(user);
}
protected boolean isMultiStructQuery() {
Set<String> filterBizName = new HashSet<>();
parseInfo.getDimensionFilters().stream()
.filter(filter -> filter.getElementID() != null)
.forEach(filter -> filterBizName.add(filter.getBizName()));
return filterBizName.size() > 1;
}
@Override
protected QueryStructReq convertQueryStruct() {
QueryStructReq queryStructReq = super.convertQueryStruct();
addDimension(queryStructReq, true);
return queryStructReq;
}
@Override
protected QueryMultiStructReq convertQueryMultiStruct() {
QueryMultiStructReq queryMultiStructReq = super.convertQueryMultiStruct();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
addDimension(queryStructReq, false);
}
return queryMultiStructReq;
}
private void addDimension(QueryStructReq queryStructReq, boolean onlyOperateInFilter) {
if (!queryStructReq.getDimensionFilters().isEmpty()) {
List<String> dimensions = queryStructReq.getGroups();
log.info("addDimension before [{}]", queryStructReq.getGroups());
List<Filter> filters = new ArrayList<>(queryStructReq.getDimensionFilters());
if (onlyOperateInFilter) {
filters = filters.stream().filter(filter
-> filter.getOperator().equals(FilterOperatorEnum.IN)).collect(Collectors.toList());
}
filters.forEach(d -> {
if (!dimensions.contains(d.getBizName())) {
dimensions.add(d.getBizName());
}});
queryStructReq.setGroups(dimensions);
log.info("addDimension after [{}]", queryStructReq.getGroups());
}
}
}

31
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java
View File

@@ -1,23 +1,22 @@
package com.tencent.supersonic.chat.query.rule.metric;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL;
@Slf4j
@Component
@@ -27,8 +26,7 @@ public class MetricFilterQuery extends MetricSemanticQuery {
public MetricFilterQuery() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
}
@Override
@@ -40,14 +38,7 @@ public class MetricFilterQuery extends MetricSemanticQuery {
public QueryResult execute(User user) {
if (!isMultiStructQuery()) {
QueryResult queryResult = super.execute(user);
if (Objects.nonNull(queryResult)) {
QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp();
queryResp.setColumns(queryResult.getQueryColumns());
queryResp.setResultList(queryResult.getQueryResults());
AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class)
.getAggregateInfo(user,parseInfo,queryResp);
queryResult.setAggregateInfo(aggregateInfo);
}
fillAggregateInfo(user, queryResult);
return queryResult;
}
return super.multiStructExecute(user);

57
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -1,27 +1,32 @@
package com.tencent.supersonic.chat.query.rule.metric;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
@@ -78,23 +83,30 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
}
@Override
public void fillParseInfo(Long domainId, ChatContext chatContext){
public void fillParseInfo(Long domainId, ChatContext chatContext) {
super.fillParseInfo(domainId, chatContext);
parseInfo.setLimit(METRIC_MAX_RESULTS);
if (parseInfo.getDateInfo() == null) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRich chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId());
ChatDefaultRichConfig defaultConfig = chatConfig.getChatAggRichConfig().getChatDefaultConfig();
ChatConfigRichResp chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId());
ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatAggRichConfig().getChatDefaultConfig();
DateConf dateInfo = new DateConf();
int unit = 1;
if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) {
unit = defaultConfig.getUnit();
}
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = LocalDate.now().plusDays(-1).toString();
DateConf dateInfo = new DateConf();
dateInfo.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS);
String endDate = startDate;
if (ChatDefaultConfigReq.TimeMode.LAST.equals(defaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
} else if (ChatDefaultConfigReq.TimeMode.RECENT.equals(defaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.RECENT);
endDate = LocalDate.now().plusDays(-1).toString();
}
dateInfo.setUnit(unit);
dateInfo.setPeriod(defaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
@@ -102,4 +114,15 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
}
}
public void fillAggregateInfo(User user, QueryResult queryResult) {
if (Objects.nonNull(queryResult)) {
QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp();
queryResp.setColumns(queryResult.getQueryColumns());
queryResp.setResultList(queryResult.getQueryResults());
AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class)
.getAggregateInfo(user, parseInfo, queryResp);
queryResult.setAggregateInfo(aggregateInfo);
}
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java
View File

@@ -54,7 +54,7 @@ public class MetricTopNQuery extends MetricSemanticQuery {
super.fillParseInfo(domainId, chatContext);
parseInfo.setLimit(ORDERBY_MAX_RESULTS);
parseInfo.setBonus(2.0);
parseInfo.setScore(2.0);
parseInfo.setAggType(AggregateTypeEnum.SUM);
SchemaElement metric = parseInfo.getMetrics().iterator().next();

10
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java
View File

@@ -4,7 +4,11 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
@@ -65,12 +69,12 @@ public class ChatConfigController {
@GetMapping("/richDesc/{domainId}")
public ChatConfigRich getDomainExtendRichInfo(@PathVariable("domainId") Long domainId) {
public ChatConfigRichResp getDomainExtendRichInfo(@PathVariable("domainId") Long domainId) {
return configService.getConfigRichInfo(domainId);
}
@GetMapping("/richDesc/all")
public List<ChatConfigRich> getAllChatRichConfig() {
public List<ChatConfigRichResp> getAllChatRichConfig() {
return configService.getAllChatRichConfig();
}

10
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java
View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.service.ChatService;
import java.util.List;
@@ -68,10 +68,10 @@ public class ChatController {
}
@PostMapping("/pageQueryInfo")
public PageInfo<QueryResponse> pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand,
@RequestParam(value = "chatId") long chatId,
HttpServletRequest request,
HttpServletResponse response) {
public PageInfo<QueryResp> pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand,
@RequestParam(value = "chatId") long chatId,
HttpServletRequest request,
HttpServletResponse response) {
pageQueryInfoCommand.setUserName(UserHolder.findUser(request, response).getName());
return chatService.queryInfo(pageQueryInfoCommand, chatId);
}

27
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java
View File

@@ -2,8 +2,9 @@ package com.tencent.supersonic.chat.rest;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.chat.service.SearchService;
import javax.servlet.http.HttpServletRequest;
@@ -31,28 +32,42 @@ public class ChatQueryController {
@PostMapping("search")
public Object search(@RequestBody QueryRequest queryCtx, HttpServletRequest request,
public Object search(@RequestBody QueryReq queryCtx, HttpServletRequest request,
HttpServletResponse response) {
queryCtx.setUser(UserHolder.findUser(request, response));
return searchService.search(queryCtx);
}
@PostMapping("query")
public Object query(@RequestBody QueryRequest queryCtx, HttpServletRequest request, HttpServletResponse response)
public Object query(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response)
throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.executeQuery(queryCtx);
}
@PostMapping("parse")
public Object parse(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response)
throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.performParsing(queryCtx);
}
@PostMapping("execute")
public Object execute(@RequestBody ExecuteQueryReq queryCtx, HttpServletRequest request, HttpServletResponse response)
throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.performExecution(queryCtx);
}
@PostMapping("queryContext")
public Object queryContext(@RequestBody QueryRequest queryCtx, HttpServletRequest request,
public Object queryContext(@RequestBody QueryReq queryCtx, HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.queryContext(queryCtx);
}
@PostMapping("queryData")
public Object queryData(@RequestBody QueryDataRequest queryData, HttpServletRequest request, HttpServletResponse response)
public Object queryData(@RequestBody QueryDataReq queryData, HttpServletRequest request, HttpServletResponse response)
throws Exception {
return queryService.executeDirectQuery(queryData, UserHolder.findUser(request, response));
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/rest/PluginController.java
View File

@@ -50,7 +50,7 @@ public class PluginController {
}
@PostMapping("/query")
List<Plugin> query(PluginQueryReq pluginQueryReq) {
List<Plugin> query(@RequestBody PluginQueryReq pluginQueryReq) {
return pluginService.queryWithAuthCheck(pluginQueryReq);
}

32
chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.rest;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import com.tencent.supersonic.chat.service.RecommendService;
import javax.servlet.http.HttpServletRequest;
@@ -25,31 +25,31 @@ public class RecommendController {
private RecommendService recommendService;
@GetMapping("recommend/{domainId}")
public RecommendResponse recommend(@PathVariable("domainId") Long domainId,
@RequestParam(value = "limit", required = false) Long limit,
HttpServletRequest request,
HttpServletResponse response) {
QueryRequest queryCtx = new QueryRequest();
public RecommendResp recommend(@PathVariable("domainId") Long domainId,
@RequestParam(value = "limit", required = false) Long limit,
HttpServletRequest request,
HttpServletResponse response) {
QueryReq queryCtx = new QueryReq();
queryCtx.setUser(UserHolder.findUser(request, response));
queryCtx.setDomainId(domainId);
return recommendService.recommend(queryCtx, limit);
}
@GetMapping("recommend/metric/{domainId}")
public RecommendResponse recommendMetricMode(@PathVariable("domainId") Long domainId,
@RequestParam(value = "limit", required = false) Long limit,
HttpServletRequest request,
HttpServletResponse response) {
QueryRequest queryCtx = new QueryRequest();
public RecommendResp recommendMetricMode(@PathVariable("domainId") Long domainId,
@RequestParam(value = "limit", required = false) Long limit,
HttpServletRequest request,
HttpServletResponse response) {
QueryReq queryCtx = new QueryReq();
queryCtx.setUser(UserHolder.findUser(request, response));
queryCtx.setDomainId(domainId);
return recommendService.recommendMetricMode(queryCtx, limit);
}
@GetMapping("recommend/question")
public List<RecommendQuestion> recommendQuestion(@RequestParam(value = "domainId", required = false) Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
public List<RecommendQuestionResp> recommendQuestion(@RequestParam(value = "domainId", required = false) Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
return recommendService.recommendQuestion(domainId);
}
}

8
chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import java.util.List;
@@ -25,8 +25,6 @@ public interface ChatService {
public void updateContext(ChatContext chatCtx);
public void updateContext(ChatContext chatCtx, QueryContext queryCtx, SemanticParseInfo semanticParseInfo);
public void switchContext(ChatContext chatCtx);
public Boolean addChat(User user, String chatName);
@@ -41,9 +39,9 @@ public interface ChatService {
Boolean deleteChat(Long chatId, String userName);
PageInfo<QueryResponse> queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId);
PageInfo<QueryResp> queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId);
public void addQuery(QueryResult queryResult, QueryContext queryContext, ChatContext chatCtx);
public void addQuery(QueryResult queryResult, ChatContext chatCtx);
public ChatQueryDO getLastQuery(long chatId);

10
chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java
View File

@@ -2,7 +2,11 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import java.util.List;
@@ -14,9 +18,9 @@ public interface ConfigService {
List<ChatConfigResp> search(ChatConfigFilter filter, User user);
ChatConfigRich getConfigRichInfo(Long domainId);
ChatConfigRichResp getConfigRichInfo(Long domainId);
ChatConfigResp fetchConfigByDomainId(Long domainId);
List<ChatConfigRich> getAllChatRichConfig();
List<ChatConfigRichResp> getAllChatRichConfig();
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java
View File

@@ -2,9 +2,11 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import org.apache.calcite.sql.parser.SqlParseException;
/***
@@ -12,9 +14,14 @@ import org.apache.calcite.sql.parser.SqlParseException;
*/
public interface QueryService {
QueryResult executeQuery(QueryRequest queryCtx) throws Exception;
ParseResp performParsing(QueryReq queryReq);
SemanticParseInfo queryContext(QueryRequest queryCtx);
QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception;
QueryResult executeQuery(QueryReq queryReq) throws Exception;
SemanticParseInfo queryContext(QueryReq queryReq);
QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException;
QueryResult executeDirectQuery(QueryDataRequest queryData, User user) throws SqlParseException;
}

12
chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import java.util.List;
@@ -12,9 +12,9 @@ import java.util.List;
*/
public interface RecommendService {
RecommendResponse recommend(QueryRequest queryCtx, Long limit);
RecommendResp recommend(QueryReq queryCtx, Long limit);
RecommendResponse recommendMetricMode(QueryRequest queryCtx, Long limit);
RecommendResp recommendMetricMode(QueryReq queryCtx, Long limit);
List<RecommendQuestion> recommendQuestion(Long domainId);
List<RecommendQuestionResp> recommendQuestion(Long domainId);
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/service/SearchService.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SearchResult;
import java.util.List;
@@ -9,6 +9,6 @@ import java.util.List;
*/
public interface SearchService {
List<SearchResult> search(QueryRequest queryCtx);
List<SearchResult> search(QueryReq queryCtx);
}

141
chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java
View File

@@ -16,19 +16,20 @@ import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.DataInfo;
import com.tencent.supersonic.chat.api.pojo.response.DomainInfo;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
import com.tencent.supersonic.chat.config.ChatAggConfig;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.config.ChatDetailConfig;
import com.tencent.supersonic.chat.config.EntityRichInfo;
import com.tencent.supersonic.chat.config.ItemVisibility;
import com.tencent.supersonic.chat.config.AggregatorConfig;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.DateConf;
@@ -36,11 +37,13 @@ import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.RatioOverType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.text.DecimalFormat;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -56,6 +59,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -70,6 +74,8 @@ public class SemanticService {
private SchemaService schemaService;
@Autowired
private ConfigService configService;
@Autowired
private AggregatorConfig aggregatorConfig;
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@@ -122,25 +128,30 @@ public class SemanticService {
}
public EntityInfo getEntityInfo(Long domain) {
ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo(domain);
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain);
if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) {
return new EntityInfo();
}
return getEntityInfo(chaConfigRichDesc);
}
private EntityInfo getEntityInfo(ChatConfigRich chaConfigRichDesc) {
private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) {
EntityInfo entityInfo = new EntityInfo();
EntityRichInfo entityDesc = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
if (entityDesc != null && Objects.nonNull(chaConfigRichDesc.getDomainId())) {
Long domainId = chaConfigRichDesc.getDomainId();
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(domainId)) {
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = schemaService.getDomainSchema(domainId);
if (Objects.isNull(domainSchema) || Objects.isNull(domainSchema.getEntity())) {
return entityInfo;
}
DomainInfo domainInfo = new DomainInfo();
domainInfo.setItemId(chaConfigRichDesc.getDomainId().intValue());
domainInfo.setName(chaConfigRichDesc.getDomainName());
domainInfo.setWords(entityDesc.getNames());
domainInfo.setBizName(chaConfigRichDesc.getBizName());
if (Objects.nonNull(entityDesc.getDimItem())) {
domainInfo.setPrimaryEntityBizName(entityDesc.getDimItem().getBizName());
domainInfo.setItemId(domainId.intValue());
domainInfo.setName(domainSchema.getDomain().getName());
domainInfo.setWords(domainSchema.getDomain().getAlias());
domainInfo.setBizName(domainSchema.getDomain().getBizName());
if (Objects.nonNull(domainSchema.getEntity())) {
domainInfo.setPrimaryEntityBizName(domainSchema.getEntity().getBizName());
}
entityInfo.setDomainInfo(domainInfo);
@@ -149,7 +160,7 @@ public class SemanticService {
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())
&& Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) {
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig()
ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig()
.getChatDefaultConfig();
if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())) {
for (SchemaElement dimensionDesc : chatDefaultConfig.getDimensions()) {
@@ -187,8 +198,21 @@ public class SemanticService {
semanticParseInfo.setMetrics(getMetrics(domainInfo));
semanticParseInfo.setDimensions(getDimensions(domainInfo));
DateConf dateInfo = new DateConf();
dateInfo.setUnit(1);
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
int unit = 1;
ChatConfigResp chatConfigInfo =
configService.fetchConfigByDomainId(domainSchema.getDomain().getId());
if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(chatConfigInfo.getChatDetailConfig())
&& Objects.nonNull(chatConfigInfo.getChatDetailConfig().getChatDefaultConfig())) {
ChatDefaultConfigReq chatDefaultConfig = chatConfigInfo.getChatDetailConfig().getChatDefaultConfig();
unit = chatDefaultConfig.getUnit();
String date = LocalDate.now().plusDays(-unit).toString();
dateInfo.setDateMode(DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);
} else {
dateInfo.setUnit(unit);
dateInfo.setDateMode(DateMode.RECENT);
}
semanticParseInfo.setDateInfo(dateInfo);
// add filter
@@ -286,8 +310,8 @@ public class SemanticService {
private ItemVisibility generateFinalVisibility(ChatConfigResp chatConfigInfo) {
ItemVisibility visibility = new ItemVisibility();
ChatAggConfig chatAggConfig = chatConfigInfo.getChatAggConfig();
ChatDetailConfig chatDetailConfig = chatConfigInfo.getChatDetailConfig();
ChatAggConfigReq chatAggConfig = chatConfigInfo.getChatAggConfig();
ChatDetailConfigReq chatDetailConfig = chatConfigInfo.getChatDetailConfig();
// both black is exist
if (Objects.nonNull(chatAggConfig) && Objects.nonNull(chatAggConfig.getVisibility())
@@ -307,8 +331,8 @@ public class SemanticService {
}
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp result) {
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) {
QueryResultWithSchemaResp result) {
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) {
return new AggregateInfo();
}
List<String> resultMetricNames = result.getColumns().stream().map(c -> c.getNameEn())
@@ -319,24 +343,36 @@ public class SemanticService {
AggregateInfo aggregateInfo = new AggregateInfo();
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp = result.getResultList().stream()
.map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder()).findFirst();
if (lastDayOp.isPresent()) {
Optional<Map<String, Object>> lastValue = result.getResultList().stream()
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
if (lastValue.isPresent()) {
metricInfo.setValue(lastValue.get().get(ratioMetric.get().getBizName()).toString());
}
metricInfo.setDate(lastValue.get().get(dateField).toString());
}
try {
queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL,
result, metricInfo);
queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER,
result, metricInfo);
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp = result.getResultList().stream().filter(r -> r.containsKey(dateField))
.map(r -> r.get(dateField).toString())
.sorted(Comparator.reverseOrder()).findFirst();
if (lastDayOp.isPresent()) {
Optional<Map<String, Object>> lastValue = result.getResultList().stream()
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) {
DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName())));
}
metricInfo.setDate(lastValue.get().get(dateField).toString());
}
CompletableFuture<MetricInfo> metricInfoRoll = CompletableFuture
.supplyAsync(() -> {
return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL,
result);
});
CompletableFuture<MetricInfo> metricInfoOver = CompletableFuture
.supplyAsync(() -> {
return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER,
result);
});
CompletableFuture.allOf(metricInfoRoll, metricInfoOver);
metricInfo.setName(metricInfoRoll.get().getName());
metricInfo.setValue(metricInfoRoll.get().getValue());
metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
aggregateInfo.getMetricInfos().add(metricInfo);
} catch (Exception e) {
log.error("queryRatio error {}", e);
@@ -346,8 +382,10 @@ public class SemanticService {
return new AggregateInfo();
}
private void queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results, MetricInfo metricInfo) {
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) {
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
DateConf dateInfo = semanticParseInfo.getDateInfo();
String dateField = QueryReqBuilder.getDateField(dateInfo);
@@ -360,12 +398,18 @@ public class SemanticService {
Map<String, Object> result = queryResp.getResultList().get(0);
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
if (valueColumn.isPresent()) {
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(),
aggOperatorEnum.getOperator());
if (result.containsKey(valueColumn.get().getNameEn())) {
DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
}
String ratio = "";
if (Objects.nonNull(result.get(valueColumn.get().getNameEn()))) {
if (Objects.nonNull(result.get(valueField))) {
ratio = String.format("%.2f",
(Double.valueOf(result.get(valueColumn.get().getNameEn()).toString()) * 100)) + "%";
(Double.valueOf(result.get(valueField).toString()) * 100)) + "%";
}
String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName();
String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName();
@@ -383,10 +427,11 @@ public class SemanticService {
}
metricInfo.setName(metric.getName());
}
return metricInfo;
}
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp results) {
QueryResultWithSchemaResp results) {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp = results.getResultList().stream()
.map(r -> r.get(dateField).toString())
@@ -395,7 +440,7 @@ public class SemanticService {
String lastDay = lastDayOp.get();
DateConf dateConf = new DateConf();
dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod());
dateConf.setDateMode(DateMode.LIST_DISCRETE);
dateConf.setDateMode(DateMode.LIST);
List<String> dayList = new ArrayList<>();
dayList.add(lastDay);
String start = "";

18
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.QueryDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository;
import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
@@ -68,14 +68,6 @@ public class ChatServiceImpl implements ChatService {
chatContextRepository.updateContext(chatCtx);
}
@Override
public void updateContext(ChatContext chatCtx, QueryContext queryCtx,
SemanticParseInfo semanticParseInfo) {
chatCtx.setParseInfo(semanticParseInfo);
chatCtx.setQueryText(queryCtx.getRequest().getQueryText());
updateContext(chatCtx);
}
@Override
public void switchContext(ChatContext chatCtx) {
log.debug("switchContext ChatContext {}", chatCtx);
@@ -126,15 +118,15 @@ public class ChatServiceImpl implements ChatService {
}
@Override
public PageInfo<QueryResponse> queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId) {
public PageInfo<QueryResp> queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId) {
return chatQueryRepository.getChatQuery(pageQueryInfoCommend, chatId);
}
@Override
public void addQuery(QueryResult queryResult, QueryContext queryContext, ChatContext chatCtx) {
chatQueryRepository.createChatQuery(queryResult, queryContext.getRequest(), chatCtx);
public void addQuery(QueryResult queryResult, ChatContext chatCtx) {
chatQueryRepository.createChatQuery(queryResult, chatCtx);
chatRepository.updateLastQuestion(chatCtx.getChatId().longValue(),
queryContext.getRequest().getQueryText(), getCurrentTime());
chatCtx.getQueryText(), getCurrentTime());
}
@Override

37
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java
View File

@@ -5,6 +5,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.*;
import com.tencent.supersonic.chat.api.pojo.response.*;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService;
@@ -135,8 +137,8 @@ public class ConfigServiceImpl implements ConfigService {
}
@Override
public ChatConfigRich getConfigRichInfo(Long domainId) {
ChatConfigRich chatConfigRich = new ChatConfigRich();
public ChatConfigRichResp getConfigRichInfo(Long domainId) {
ChatConfigRichResp chatConfigRich = new ChatConfigRichResp();
ChatConfigResp chatConfigResp = chatConfigRepository.getConfigByDomainId(domainId);
if (Objects.isNull(chatConfigResp)) {
log.info("there is no chatConfigDesc for domainId:{}", domainId);
@@ -154,24 +156,23 @@ public class ConfigServiceImpl implements ConfigService {
return chatConfigRich;
}
private ChatDetailRichConfig fillChatDetailRichConfig(DomainSchema domainSchema, ChatConfigRich chatConfigRich, ChatConfigResp chatConfigResp) {
private ChatDetailRichConfigResp fillChatDetailRichConfig(DomainSchema domainSchema, ChatConfigRichResp chatConfigRich, ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) {
return null;
}
ChatDetailRichConfig detailRichConfig = new ChatDetailRichConfig();
ChatDetailConfig chatDetailConfig = chatConfigResp.getChatDetailConfig();
ChatDetailRichConfigResp detailRichConfig = new ChatDetailRichConfigResp();
ChatDetailConfigReq chatDetailConfig = chatConfigResp.getChatDetailConfig();
ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig(chatDetailConfig.getVisibility(), domainSchema);
detailRichConfig.setVisibility(itemVisibilityInfo);
detailRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), domainSchema));
detailRichConfig.setGlobalKnowledgeConfig(chatDetailConfig.getGlobalKnowledgeConfig());
detailRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatDetailConfig.getChatDefaultConfig(), domainSchema, itemVisibilityInfo));
detailRichConfig.setEntity(generateRichEntity(chatDetailConfig.getEntity(), domainSchema));
return detailRichConfig;
}
private EntityRichInfo generateRichEntity(Entity entity, DomainSchema domainSchema) {
EntityRichInfo entityRichInfo = new EntityRichInfo();
private EntityRichInfoResp generateRichEntity(Entity entity, DomainSchema domainSchema) {
EntityRichInfoResp entityRichInfo = new EntityRichInfoResp();
if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
return entityRichInfo;
}
@@ -183,12 +184,12 @@ public class ConfigServiceImpl implements ConfigService {
return entityRichInfo;
}
private ChatAggRichConfig fillChatAggRichConfig(DomainSchema domainSchema, ChatConfigResp chatConfigResp) {
private ChatAggRichConfigResp fillChatAggRichConfig(DomainSchema domainSchema, ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
return null;
}
ChatAggConfig chatAggConfig = chatConfigResp.getChatAggConfig();
ChatAggRichConfig chatAggRichConfig = new ChatAggRichConfig();
ChatAggConfigReq chatAggConfig = chatConfigResp.getChatAggConfig();
ChatAggRichConfigResp chatAggRichConfig = new ChatAggRichConfigResp();
ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig(chatAggConfig.getVisibility(), domainSchema);
chatAggRichConfig.setVisibility(itemVisibilityInfo);
chatAggRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), domainSchema));
@@ -198,8 +199,8 @@ public class ConfigServiceImpl implements ConfigService {
return chatAggRichConfig;
}
private ChatDefaultRichConfig fetchDefaultConfig(ChatDefaultConfig chatDefaultConfig, DomainSchema domainSchema, ItemVisibilityInfo itemVisibilityInfo) {
ChatDefaultRichConfig defaultRichConfig = new ChatDefaultRichConfig();
private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig, DomainSchema domainSchema, ItemVisibilityInfo itemVisibilityInfo) {
ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp();
if (Objects.isNull(chatDefaultConfig)) {
return defaultRichConfig;
}
@@ -245,8 +246,8 @@ public class ConfigServiceImpl implements ConfigService {
}
private List<KnowledgeInfo> fillKnowledgeBizName(List<KnowledgeInfo> knowledgeInfos,
DomainSchema domainSchema) {
private List<KnowledgeInfoReq> fillKnowledgeBizName(List<KnowledgeInfoReq> knowledgeInfos,
DomainSchema domainSchema) {
if (CollectionUtils.isEmpty(knowledgeInfos)) {
return new ArrayList<>();
}
@@ -264,11 +265,11 @@ public class ConfigServiceImpl implements ConfigService {
}
@Override
public List<ChatConfigRich> getAllChatRichConfig() {
List<ChatConfigRich> chatConfigRichInfoList = new ArrayList<>();
public List<ChatConfigRichResp> getAllChatRichConfig() {
List<ChatConfigRichResp> chatConfigRichInfoList = new ArrayList<>();
List<DomainResp> domainRespList = semanticLayer.getDomainListForAdmin();
domainRespList.stream().forEach(domainResp -> {
ChatConfigRich chatConfigRichInfo = getConfigRichInfo(domainResp.getId());
ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(domainResp.getId());
if (Objects.nonNull(chatConfigRichInfo)) {
chatConfigRichInfoList.add(chatConfigRichInfo);
}

49
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDO;
@@ -14,19 +15,20 @@ import com.tencent.supersonic.chat.plugin.event.PluginDelEvent;
import com.tencent.supersonic.chat.plugin.event.PluginUpdateEvent;
import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;
@Service
@Slf4j
public class PluginServiceImpl implements PluginService {
private PluginRepository pluginRepository;
@@ -40,10 +42,12 @@ public class PluginServiceImpl implements PluginService {
}
@Override
public void createPlugin(Plugin plugin, User user){
public synchronized void createPlugin(Plugin plugin, User user){
PluginDO pluginDO = convert(plugin, user);
pluginRepository.createPlugin(pluginDO);
publisher.publishEvent(new PluginAddEvent(this, plugin));
//compatible with H2 db
List<Plugin> plugins = getPluginList();
publisher.publishEvent(new PluginAddEvent(this, plugins.get(plugins.size()-1)));
}
@Override
@@ -93,6 +97,18 @@ public class PluginServiceImpl implements PluginService {
if (StringUtils.isNotBlank(pluginQueryReq.getDomain())) {
pluginDOExample.getOredCriteria().get(0).andDomainLike('%' + pluginQueryReq.getDomain() + '%');
}
if (StringUtils.isNotBlank(pluginQueryReq.getParseMode())) {
pluginDOExample.getOredCriteria().get(0).andParseModeEqualTo(pluginQueryReq.getParseMode());
}
if (StringUtils.isNotBlank(pluginQueryReq.getName())) {
pluginDOExample.getOredCriteria().get(0).andNameLike('%' + pluginQueryReq.getName() + '%');
}
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
pluginDOExample.getOredCriteria().get(0).andPatternLike('%' + pluginQueryReq.getPattern() + '%');
}
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
pluginDOExample.getOredCriteria().get(0).andCreatedByEqualTo(pluginQueryReq.getCreatedBy());
}
List<PluginDO> pluginDOS = pluginRepository.query(pluginDOExample);
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
pluginDOS = pluginDOS.stream().filter(pluginDO ->
@@ -105,8 +121,18 @@ public class PluginServiceImpl implements PluginService {
@Override
public Optional<Plugin> getPluginByName(String name) {
log.info("name:{}", name);
return getPluginList().stream()
.filter(plugin -> plugin.getName().equalsIgnoreCase(name))
.filter(plugin -> {
if (StringUtils.isBlank(plugin.getParseModeConfig())) {
return false;
}
PluginParseConfig functionCallConfig = JsonUtil.toObject(plugin.getParseModeConfig(), PluginParseConfig.class);
if (Objects.isNull(functionCallConfig)) {
return false;
}
return functionCallConfig.getName().equalsIgnoreCase(name);
})
.findFirst();
}
@@ -120,8 +146,8 @@ public class PluginServiceImpl implements PluginService {
List<Long> domainIdAuthorized = semanticLayer.getDomainListForAdmin().stream()
.map(DomainResp::getId).collect(Collectors.toList());
plugins = plugins.stream().filter(plugin -> {
if (CollectionUtils.isEmpty(plugin.getDomainList())) {
return false;
if (CollectionUtils.isEmpty(plugin.getDomainList()) || plugin.isContainsAllDomain()) {
return true;
}
for (Long domainId : plugin.getDomainList()) {
if (domainIdAuthorized.contains(domainId)) {
@@ -136,8 +162,7 @@ public class PluginServiceImpl implements PluginService {
public Plugin convert(PluginDO pluginDO){
Plugin plugin = new Plugin();
BeanUtils.copyProperties(pluginDO,plugin);
plugin.setParseMode(ParseMode.valueOf(pluginDO.getParseMode()));
if (pluginDO.getDomain() != null) {
if (StringUtils.isNotBlank(pluginDO.getDomain())) {
plugin.setDomainList(Arrays.stream(pluginDO.getDomain().split(","))
.map(Long::parseLong).collect(Collectors.toList()));
}
@@ -152,7 +177,6 @@ public class PluginServiceImpl implements PluginService {
pluginDO.setUpdatedAt(new Date());
pluginDO.setUpdatedBy(user.getName());
pluginDO.setDomain(StringUtils.join(plugin.getDomainList(), ","));
pluginDO.setParseMode(plugin.getParseMode().name());
return pluginDO;
}
@@ -161,7 +185,6 @@ public class PluginServiceImpl implements PluginService {
pluginDO.setUpdatedAt(new Date());
pluginDO.setUpdatedBy(user.getName());
pluginDO.setDomain(StringUtils.join(plugin.getDomainList(), ","));
pluginDO.setParseMode(plugin.getParseMode().name());
return pluginDO;
}

101
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -6,15 +6,19 @@ import com.tencent.supersonic.chat.api.component.*;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@@ -41,7 +45,82 @@ public class QueryServiceImpl implements QueryService {
private QuerySelector querySelector = ComponentFactory.getQuerySelector();
@Override
public QueryResult executeQuery(QueryRequest queryReq) throws Exception {
public ParseResp performParsing(QueryReq queryReq) {
QueryContext queryCtx = new QueryContext(queryReq);
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
schemaMappers.stream().forEach(mapper -> {
mapper.map(queryCtx);
log.info("{} result:{}", mapper.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
semanticParsers.stream().forEach(parser -> {
parser.parse(queryCtx, chatCtx);
log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
ParseResp parseResult;
if (queryCtx.getCandidateQueries().size() > 0) {
log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
Collectors.toList()));
List<SemanticQuery> selectedQueries = querySelector.select(queryCtx.getCandidateQueries());
log.debug("pick after [{}]", selectedQueries.stream().collect(
Collectors.toList()));
List<SemanticParseInfo> selectedParses = selectedQueries.stream()
.map(q -> q.getParseInfo()).collect(Collectors.toList());
List<SemanticParseInfo> candidateParses = queryCtx.getCandidateQueries().stream()
.map(q -> q.getParseInfo()).collect(Collectors.toList());
parseResult = ParseResp.builder()
.chatId(queryReq.getChatId())
.queryText(queryReq.getQueryText())
.state(selectedParses.size() > 1 ? ParseResp.ParseState.PENDING : ParseResp.ParseState.COMPLETED)
.selectedParses(selectedParses)
.candidateParses(candidateParses)
.build();
} else {
parseResult = ParseResp.builder()
.chatId(queryReq.getChatId())
.queryText(queryReq.getQueryText())
.state(ParseResp.ParseState.FAILED)
.build();
}
return parseResult;
}
@Override
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
SemanticParseInfo parseInfo = queryReq.getParseInfo();
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (semanticQuery == null) {
return null;
}
semanticQuery.setParseInfo(parseInfo);
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
QueryResult queryResult = semanticQuery.execute(queryReq.getUser());
if (queryResult != null) {
queryResult.setChatContext(parseInfo);
// update chat context after a successful semantic query
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
chatCtx.setParseInfo(parseInfo);
chatService.updateContext(chatCtx);
}
chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName());
chatService.addQuery(queryResult, chatCtx);
}
return queryResult;
}
@Override
public QueryResult executeQuery(QueryReq queryReq) throws Exception {
QueryContext queryCtx = new QueryContext(queryReq);
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
@@ -60,17 +139,21 @@ public class QueryServiceImpl implements QueryService {
if (queryCtx.getCandidateQueries().size() > 0) {
log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
Collectors.toList()));
SemanticQuery semanticQuery = querySelector.select(queryCtx.getCandidateQueries());
log.info("pick after [{}]", semanticQuery);
List<SemanticQuery> selectedQueries = querySelector.select(queryCtx.getCandidateQueries());
log.info("pick after [{}]", selectedQueries.stream().collect(
Collectors.toList()));
SemanticQuery semanticQuery = selectedQueries.get(0);
queryResult = semanticQuery.execute(queryReq.getUser());
if (queryResult != null) {
chatCtx.setQueryText(queryReq.getQueryText());
// update chat context after a successful semantic query
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
chatService.updateContext(chatCtx, queryCtx, semanticQuery.getParseInfo());
chatCtx.setParseInfo(semanticQuery.getParseInfo());
chatService.updateContext(chatCtx);
}
queryResult.setChatContext(chatCtx.getParseInfo());
chatService.addQuery(queryResult, queryCtx, chatCtx);
chatService.addQuery(queryResult, chatCtx);
}
}
@@ -78,13 +161,13 @@ public class QueryServiceImpl implements QueryService {
}
@Override
public SemanticParseInfo queryContext(QueryRequest queryCtx) {
public SemanticParseInfo queryContext(QueryReq queryCtx) {
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
return context.getParseInfo();
}
@Override
public QueryResult executeDirectQuery(QueryDataRequest queryData, User user) throws SqlParseException {
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException {
SemanticQuery semanticQuery = QueryManager.createRuleQuery(queryData.getQueryMode());
BeanUtils.copyProperties(queryData, semanticQuery.getParseInfo());
return semanticQuery.execute(user);

30
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java
View File

@@ -3,12 +3,12 @@ package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion;
import com.tencent.supersonic.chat.config.ChatConfigFilter;
import com.tencent.supersonic.chat.config.ChatConfigResp;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import java.util.ArrayList;
import java.util.Comparator;
@@ -37,14 +37,14 @@ public class RecommendServiceImpl implements RecommendService {
private SemanticService semanticService;
@Override
public RecommendResponse recommend(QueryRequest queryCtx, Long limit) {
public RecommendResp recommend(QueryReq queryCtx, Long limit) {
if (Objects.isNull(limit) || limit <= 0) {
limit = Long.MAX_VALUE;
}
log.debug("limit:{}", limit);
Long domainId = queryCtx.getDomainId();
if (Objects.isNull(domainId)) {
return new RecommendResponse();
return new RecommendResp();
}
DomainSchema domainSchema = semanticService.getDomainSchema(domainId);
@@ -75,21 +75,21 @@ public class RecommendServiceImpl implements RecommendService {
return item;
}).collect(Collectors.toList());
RecommendResponse response = new RecommendResponse();
RecommendResp response = new RecommendResp();
response.setDimensions(dimensions);
response.setMetrics(metrics);
return response;
}
@Override
public RecommendResponse recommendMetricMode(QueryRequest queryCtx, Long limit) {
RecommendResponse recommendResponse = recommend(queryCtx, limit);
public RecommendResp recommendMetricMode(QueryReq queryCtx, Long limit) {
RecommendResp recommendResponse = recommend(queryCtx, limit);
// filter black Item
if (Objects.isNull(recommendResponse)) {
return recommendResponse;
}
ChatConfigRich chatConfigRich = configService.getConfigRichInfo(Long.valueOf(queryCtx.getDomainId()));
ChatConfigRichResp chatConfigRich = configService.getConfigRichInfo(Long.valueOf(queryCtx.getDomainId()));
if (Objects.nonNull(chatConfigRich) && Objects.nonNull(chatConfigRich.getChatAggRichConfig())
&& Objects.nonNull(chatConfigRich.getChatAggRichConfig().getVisibility())) {
List<Long> blackMetricIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackMetricIdList();
@@ -105,15 +105,15 @@ public class RecommendServiceImpl implements RecommendService {
}
@Override
public List<RecommendQuestion> recommendQuestion(Long domainId) {
List<RecommendQuestion> recommendQuestions = new ArrayList<>();
public List<RecommendQuestionResp> recommendQuestion(Long domainId) {
List<RecommendQuestionResp> recommendQuestions = new ArrayList<>();
ChatConfigFilter chatConfigFilter = new ChatConfigFilter();
chatConfigFilter.setDomainId(domainId);
List<ChatConfigResp> chatConfigRespList = configService.search(chatConfigFilter, null);
if (!CollectionUtils.isEmpty(chatConfigRespList)) {
chatConfigRespList.stream().forEach(chatConfigResp -> {
if (Objects.nonNull(chatConfigResp) && !CollectionUtils.isEmpty(chatConfigResp.getRecommendedQuestions())) {
recommendQuestions.add(new RecommendQuestion(chatConfigResp.getDomainId(), chatConfigResp.getRecommendedQuestions()));
recommendQuestions.add(new RecommendQuestionResp(chatConfigResp.getDomainId(), chatConfigResp.getRecommendedQuestions()));
}
});
return recommendQuestions;

8
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SearchResult;
import com.tencent.supersonic.chat.mapper.DomainInfoStat;
import com.tencent.supersonic.chat.mapper.DomainWithSemanticType;
@@ -55,7 +55,7 @@ public class SearchServiceImpl implements SearchService {
private SearchMatchStrategy searchMatchStrategy;
@Override
public List<SearchResult> search(QueryRequest queryCtx) {
public List<SearchResult> search(QueryReq queryCtx) {
String queryText = queryCtx.getQueryText();
// 1.get meta info
SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema();
@@ -109,8 +109,8 @@ public class SearchServiceImpl implements SearchService {
return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
}
private List<Long> getPossibleDomains(QueryRequest queryCtx, List<Term> originals,
DomainInfoStat domainStat, Long webDomainId) {
private List<Long> getPossibleDomains(QueryReq queryCtx, List<Term> originals,
DomainInfoStat domainStat, Long webDomainId) {
if (Objects.nonNull(webDomainId) && webDomainId > 0) {
List<Long> result = new ArrayList<>();

6
chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java
View File

@@ -14,11 +14,11 @@ public class CacheUtils {
.maximumSize(1000)
.build();
public static void put(QueryContext queryContext, ChatContext chatCtx, Object v){
String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+queryContext.getRequest().getQueryText();
String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+ queryContext.getRequest().getQueryText();
cache.put(key,v);
}
public static Object get(QueryContext queryContext,ChatContext chatCtx){
String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+queryContext.getRequest().getQueryText();
public static Object get(QueryContext queryContext, ChatContext chatCtx){
String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+ queryContext.getRequest().getQueryText();
return cache.getIfPresent(key);
}
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatConfigHelper.java
View File

@@ -5,7 +5,8 @@ import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.chat.api.pojo.request.*;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
@@ -103,9 +104,9 @@ public class ChatConfigHelper {
BeanUtils.copyProperties(chatConfigDO, chatConfigDescriptor);
chatConfigDescriptor.setChatDetailConfig(JsonUtil.toObject(chatConfigDO.getChatDetailConfig(), ChatDetailConfig.class));
chatConfigDescriptor.setChatAggConfig(JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfig.class));
chatConfigDescriptor.setRecommendedQuestions(JsonUtil.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestion.class));
chatConfigDescriptor.setChatDetailConfig(JsonUtil.toObject(chatConfigDO.getChatDetailConfig(), ChatDetailConfigReq.class));
chatConfigDescriptor.setChatAggConfig(JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfigReq.class));
chatConfigDescriptor.setRecommendedQuestions(JsonUtil.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class));
chatConfigDescriptor.setStatusEnum(StatusEnum.of(chatConfigDO.getStatus()));
chatConfigDescriptor.setCreatedBy(chatConfigDO.getCreatedBy());
@@ -132,15 +133,15 @@ public class ChatConfigHelper {
return chatConfigResp;
}
private ChatDetailConfig generateEmptyChatDetailConfigResp() {
ChatDetailConfig chatDetailConfig = new ChatDetailConfig();
private ChatDetailConfigReq generateEmptyChatDetailConfigResp() {
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ItemVisibility visibility = new ItemVisibility();
chatDetailConfig.setVisibility(visibility);
return chatDetailConfig;
}
private ChatAggConfig generateEmptyChatAggConfigResp() {
ChatAggConfig chatAggConfig = new ChatAggConfig();
private ChatAggConfigReq generateEmptyChatAggConfigResp() {
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ItemVisibility visibility = new ItemVisibility();
chatAggConfig.setVisibility(visibility);
return chatAggConfig;

78
chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatGptHelper.java Normal file
View File

@@ -0,0 +1,78 @@
package com.tencent.supersonic.chat.utils;
import com.plexpt.chatgpt.ChatGPT;
import com.plexpt.chatgpt.entity.chat.ChatCompletion;
import com.plexpt.chatgpt.entity.chat.ChatCompletionResponse;
import com.plexpt.chatgpt.entity.chat.Message;
import com.plexpt.chatgpt.util.Proxys;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.net.Proxy;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
@Component
public class ChatGptHelper {
@Value("${llm.chatgpt.apikey:sk-kdgPxxx}")
private String apiKey;
@Value("${llm.chatgpt.apiHost:https://api.openai.com/}")
private String apiHost;
@Value("${llm.chatgpt.proxyIp:default}")
private String proxyIp;
@Value("${llm.chatgpt.proxyPort:8080}")
private Integer proxyPort;
public ChatGPT getChatGPT(){
Proxy proxy = null;
if (!"default".equals(proxyIp)){
proxy = Proxys.http(proxyIp, proxyPort);
}
return ChatGPT.builder()
.apiKey(apiKey)
.proxy(proxy)
.timeout(900)
.apiHost(apiHost) //反向代理地址
.build()
.init();
}
public String inferredTime(String queryText){
long nowTime = System.currentTimeMillis();
Date date = new Date(nowTime);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String formattedDate = sdf.format(date);
Message system = Message.ofSystem("现在时间 "+formattedDate+"，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。" +
"你需要遵守以下规则：\n" +
"1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}" +
"2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n"+
"3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n" +
"4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n"+
"5.时间给出要是绝对正确，不能瞎编\n"
);
Message message = Message.of("输入："+queryText+"，输出：");
ChatCompletion chatCompletion = ChatCompletion.builder()
.model(ChatCompletion.Model.GPT_3_5_TURBO_16K.getName())
.messages(Arrays.asList(system, message))
.maxTokens(10000)
.temperature(0.9)
.build();
ChatCompletionResponse response = getChatGPT().chatCompletion(chatCompletion);
Message res = response.getChoices().get(0).getMessage();
return res.getContent();
}
public static void main(String[] args) {
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java
View File

@@ -6,6 +6,10 @@ import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
@@ -124,13 +128,13 @@ public class DictMetaHelper {
private void fillDimValueDOList(List<DimValueDO> dimValueDOList, Long domainId,
Map<Long, SchemaElement> dimIdAndDescPair) {
ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo(domainId);
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domainId);
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) {
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
List<KnowledgeInfo> knowledgeAggInfo = chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
List<KnowledgeInfoReq> knowledgeAggInfo = chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
List<KnowledgeInfo> knowledgeDetailInfo = chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
List<KnowledgeInfoReq> knowledgeDetailInfo = chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, domainId);
fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, domainId);
@@ -139,7 +143,7 @@ public class DictMetaHelper {
}
}
private void fillKnowledgeDimValue(List<KnowledgeInfo> knowledgeInfos, ChatDefaultRichConfig chatDefaultConfig,
private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos, ChatDefaultRichConfigResp chatDefaultConfig,
List<DimValueDO> dimValueDOList, Map<Long, SchemaElement> dimIdAndDescPair, Long domainId) {
if (!CollectionUtils.isEmpty(knowledgeInfos)) {
List<Dim4Dict> dimensions = new ArrayList<>();

4
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java
View File

@@ -91,7 +91,7 @@ public class DictQueryHelper {
}
private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
String metricName) {
String metricName) {
List<String> data = new ArrayList<>();
if (CollectionUtils.isEmpty(resultList)) {
return data;
@@ -160,7 +160,7 @@ public class DictQueryHelper {
queryStructCmd.setOrders(orders);
DateConf dateInfo = new DateConf();
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
dateInfo.setUnit(defaultMetricDesc.getUnit());
queryStructCmd.setDateInfo(dateInfo);

92
chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java
View File

@@ -3,9 +3,7 @@ package com.tencent.supersonic.chat.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
@@ -17,11 +15,8 @@ import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.time.LocalDate;
import java.util.*;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
@@ -35,7 +30,7 @@ public class QueryReqBuilder {
QueryStructReq queryStructCmd = new QueryStructReq();
queryStructCmd.setDomainId(parseInfo.getDomainId());
queryStructCmd.setNativeQuery(parseInfo.getNativeQuery());
queryStructCmd.setDateInfo(parseInfo.getDateInfo());
queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo()));
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
@@ -49,16 +44,47 @@ public class QueryReqBuilder {
queryStructCmd.setMetricFilters(metricFilters);
addDateDimension(parseInfo);
List<String> dimensions = parseInfo.getDimensions().stream().map(entry -> entry.getBizName())
List<String> dimensions = parseInfo.getDimensions().stream().map(SchemaElement::getBizName)
.collect(Collectors.toList());
queryStructCmd.setGroups(dimensions);
queryStructCmd.setLimit(parseInfo.getLimit());
Set<Order> order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics());
queryStructCmd.setOrders(new ArrayList<>(order));
queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getMetrics(), parseInfo.getAggType()));
// only one metric is queried at once
Set<SchemaElement> metrics = parseInfo.getMetrics();
if (!CollectionUtils.isEmpty(metrics)) {
SchemaElement metricElement = parseInfo.getMetrics().iterator().next();
Set<Order> order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), metricElement);
queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getAggType(), metricElement));
queryStructCmd.setOrders(new ArrayList<>(order));
}
deletionDuplicated(queryStructCmd);
return queryStructCmd;
}
private static void deletionDuplicated(QueryStructReq queryStructReq) {
if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) {
Set<String> groups = new HashSet<>();
groups.addAll(queryStructReq.getGroups());
queryStructReq.getGroups().clear();
queryStructReq.getGroups().addAll(groups);
}
}
private static DateConf rewrite2Between(DateConf dateInfo) {
DateConf dateInfoNew = new DateConf();
BeanUtils.copyProperties(dateInfo, dateInfoNew);
if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT.equals(dateInfo.getDateMode())) {
int unit = dateInfo.getUnit();
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = LocalDate.now().plusDays(-1).toString();
dateInfoNew.setDateMode(DateConf.DateMode.BETWEEN);
dateInfoNew.setStartDate(startDate);
dateInfoNew.setEndDate(endDate);
}
return dateInfoNew;
}
public static QueryMultiStructReq buildMultiStructReq(SemanticParseInfo parseInfo) {
QueryStructReq queryStructReq = buildStructReq(parseInfo);
QueryMultiStructReq queryMultiStructReq = new QueryMultiStructReq();
@@ -75,11 +101,12 @@ public class QueryReqBuilder {
/**
* convert to QueryDslReq
*
* @param querySql
* @param domainId
* @return
*/
public static QueryDslReq buildDslReq(String querySql,Long domainId) {
public static QueryDslReq buildDslReq(String querySql, Long domainId) {
QueryDslReq queryDslReq = new QueryDslReq();
if (Objects.nonNull(querySql)) {
queryDslReq.setSql(querySql);
@@ -89,11 +116,11 @@ public class QueryReqBuilder {
}
private static List<Aggregator> getAggregatorByMetric(Set<SchemaElement> metrics, AggregateTypeEnum aggregateType) {
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {
List<Aggregator> aggregators = new ArrayList<>();
String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? ""
: aggregateType.name();
for (SchemaElement metric : metrics) {
if(metric != null) {
String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? ""
: aggregateType.name();
aggregators.add(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg)));
}
return aggregators;
@@ -118,41 +145,42 @@ public class QueryReqBuilder {
return;
}
}
if (Objects.nonNull(parseInfo.getAggType()) && !parseInfo.getAggType().equals(AggregateTypeEnum.NONE)) {
return;
}
SchemaElement dimension = new SchemaElement();
dimension.setBizName(dateField);
if (MetricDomainQuery.QUERY_MODE.equals(queryMode)
|| MetricGroupByQuery.QUERY_MODE.equals(queryMode)
|| MetricFilterQuery.QUERY_MODE.equals(queryMode)
) {
if (QueryManager.isMetricQuery(queryMode)) {
parseInfo.getDimensions().add(dimension);
}
}
}
public static Set<Order> getOrder(Set<Order> parseOrder, AggregateTypeEnum aggregator, Set<SchemaElement> metrics) {
public static Set<Order> getOrder(Set<Order> parseOrder, AggregateTypeEnum aggregator, SchemaElement metric) {
if (!CollectionUtils.isEmpty(parseOrder)) {
return parseOrder;
}
Set<Order> orders = new LinkedHashSet();
if (CollectionUtils.isEmpty(metrics)) {
if (metric == null) {
return orders;
}
if ((AggregateTypeEnum.TOPN.equals(aggregator) || AggregateTypeEnum.MAX.equals(aggregator)
|| AggregateTypeEnum.MIN.equals(
aggregator))) {
for (SchemaElement metric : metrics) {
Order order = new Order();
order.setColumn(metric.getBizName());
order.setDirection("desc");
orders.add(order);
}
Order order = new Order();
order.setColumn(metric.getBizName());
order.setDirection("desc");
orders.add(order);
}
return orders;
}
public static String getDateField(DateConf dateConf) {
if(Objects.isNull(dateConf)) {
if (Objects.isNull(dateConf)) {
return "";
}
String dateField = TimeDimensionEnum.DAY.getName();
@@ -165,7 +193,7 @@ public class QueryReqBuilder {
return dateField;
}
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo,SchemaElement metric,AggOperatorEnum aggOperatorEnum) {
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum) {
QueryStructReq queryStructCmd = buildStructReq(parseInfo);
queryStructCmd.setNativeQuery(false);
queryStructCmd.setOrders(new ArrayList<>());

32
chat/core/src/main/resources/mapper/PluginDOMapper.xml
View File

@@ -14,7 +14,9 @@
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
<result column="parse_mode_config" jdbcType="LONGVARCHAR" property="parseModeConfig" />
<result column="config" jdbcType="LONGVARCHAR" property="config" />
<result column="comment" jdbcType="LONGVARCHAR" property="comment" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
@@ -50,7 +52,7 @@
updated_by
</sql>
<sql id="Blob_Column_List">
config
parse_mode_config, config, comment
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample" resultMap="ResultMapWithBLOBs">
select
@@ -101,11 +103,13 @@
insert into s2_plugin (id, type, domain,
pattern, parse_mode, name,
created_at, created_by, updated_at,
updated_by, config)
updated_by, parse_mode_config, config,
comment)
values (#{id,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{domain,jdbcType=VARCHAR},
#{pattern,jdbcType=VARCHAR}, #{parseMode,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{config,jdbcType=LONGVARCHAR})
#{updatedBy,jdbcType=VARCHAR}, #{parseModeConfig,jdbcType=LONGVARCHAR}, #{config,jdbcType=LONGVARCHAR},
#{comment,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
insert into s2_plugin
@@ -140,9 +144,15 @@
<if test="updatedBy != null">
updated_by,
</if>
<if test="parseModeConfig != null">
parse_mode_config,
</if>
<if test="config != null">
config,
</if>
<if test="comment != null">
comment,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
@@ -175,9 +185,15 @@
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
#{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
#{comment,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample" resultType="java.lang.Long">
@@ -216,9 +232,15 @@
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
comment = #{comment,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
@@ -233,7 +255,9 @@
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
config = #{config,jdbcType=LONGVARCHAR}
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
config = #{config,jdbcType=LONGVARCHAR},
comment = #{comment,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">

4
chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.rule.TimeRangeParser;
import org.junit.jupiter.api.Test;
@@ -27,7 +27,7 @@ class TimeRangeParserTest {
void parse() {
TimeRangeParser timeRangeParser = new TimeRangeParser();
QueryRequest queryRequest = new QueryRequest();
QueryReq queryRequest = new QueryReq();
ChatContext chatCtx = new ChatContext();
SchemaMapInfo schemaMap = new SchemaMapInfo();

4
chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.mapper;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.test.context.ContextTest;
import org.junit.jupiter.api.Test;
@@ -12,7 +12,7 @@ class HanlpDictMapperTest extends ContextTest {
@Test
void map() {
QueryRequest queryRequest = new QueryRequest();
QueryReq queryRequest = new QueryReq();
queryRequest.setChatId(1);
queryRequest.setDomainId(2L);
queryRequest.setQueryText("supersonic按部门访问次数");

7
chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java
View File

@@ -6,6 +6,9 @@ import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp;
import com.tencent.supersonic.chat.config.*;
import com.tencent.supersonic.chat.persistence.repository.impl.ChatContextRepositoryImpl;
import com.tencent.supersonic.chat.service.QueryService;
@@ -40,12 +43,12 @@ public class MockBeansConfiguration {
public static void buildHttpSemanticServiceImpl(SemanticLayer httpSemanticLayer, List<DimSchemaResp> dimensionDescs,
List<MetricSchemaResp> metricDescs) {
ChatConfigRich chaConfigRichDesc = new ChatConfigRich();
ChatConfigRichResp chaConfigRichDesc = new ChatConfigRichResp();
DefaultMetric defaultMetricDesc = new DefaultMetric();
defaultMetricDesc.setUnit(3);
defaultMetricDesc.setPeriod(Constants.DAY);
// chaConfigRichDesc.setDefaultMetrics(new ArrayList<>(Arrays.asList(defaultMetricDesc)));
EntityRichInfo entityDesc = new EntityRichInfo();
EntityRichInfoResp entityDesc = new EntityRichInfoResp();
List<DimSchemaResp> dimensionDescs1 = new ArrayList<>();
DimSchemaResp dimensionDesc = new DimSchemaResp();
dimensionDesc.setId(162L);

2
chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java
View File

@@ -66,7 +66,7 @@ public class SemanticParseObjectHelper {
if (dayAgo > 0) {
DateConf dateInfo = new DateConf();
dateInfo.setUnit(dayAgo);
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
return dateInfo;
}
return null;