topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 20:25:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) support for modifying filter conditions and fix group by pushdown and add windows scipt (#49)

Browse Source 
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
2023-09-03 23:51:47 +08:00
committed by GitHub
parent 8440f1f30e
commit 559ef974b0
317 changed files with 7449 additions and 9413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/CorrectionInfo.java
View File

@@ -18,4 +18,5 @@ public class CorrectionInfo {
private String sql;
private String preSql;
}

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ModelSchema.java
View File

@@ -1,9 +1,10 @@
package com.tencent.supersonic.chat.api.pojo;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import lombok.Data;
@Data
public class ModelSchema {

5
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java
View File

@@ -2,11 +2,12 @@ package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@NoArgsConstructor
public class QueryContext {

20
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java
View File

@@ -1,39 +1,30 @@
package com.tencent.supersonic.chat.api.pojo;
import com.google.common.base.Objects;
import java.io.Serializable;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.Builder;
import lombok.NoArgsConstructor;
@Data
@Getter
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class SchemaElement implements Serializable {
private Long model;
private Long id;
private String name;
private String bizName;
private Long useCnt;
private SchemaElementType type;
private List<String> alias;
public SchemaElement(Long model, Long id, String name, String bizName,
Long useCnt, SchemaElementType type, List<String> alias) {
this.model = model;
this.id = id;
this.name = name;
this.bizName = bizName;
this.useCnt = useCnt;
this.type = type;
this.alias = alias;
}
private List<SchemaValueMap> schemaValueMaps;
@Override
public boolean equals(Object o) {
@@ -54,4 +45,5 @@ public class SchemaElement implements Serializable {
public int hashCode() {
return Objects.hashCode(model, id, name, bizName, useCnt, type);
}
}

24
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaValueMap.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.chat.api.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class SchemaValueMap {
/**
* dimension value in db
*/
private String techName;
/**
* dimension value for result show
*/
private String bizName;
/**
* dimension value for user query
*/
private List<String> alias = new ArrayList<>();
}

1
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java
View File

@@ -7,7 +7,6 @@ import java.util.Map;
import java.util.stream.Collectors;
public class SemanticSchema implements Serializable {
private List<ModelSchema> modelSchemaList;
public SemanticSchema(List<ModelSchema> modelSchemaList) {

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggConfigReq {

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java
View File

@@ -1,10 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import java.util.List;
/**
* extended information command about model
*/

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java
View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class ChatDefaultConfigReq {

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailConfigReq {

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
@Data
public class DimensionValueReq {
private Long modelId;
private String bizName;
private Object value;
}

5
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java
View File

@@ -7,13 +7,12 @@ import lombok.Data;
@Data
public class ExecuteQueryReq {
private User user;
private Integer agentId;
private Integer chatId;
private String queryText;
private Long queryId;
private Integer parseId;
private Long queryId = 7L;
private Integer parseId = 2;
private SemanticParseInfo parseInfo;
private boolean saveAnswer = true;
}

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
/**
* advanced knowledge config

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java
View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import javax.validation.constraints.NotNull;
import lombok.Data;
/**

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java
View File

@@ -4,19 +4,20 @@ package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.HashSet;
import java.util.Set;
import lombok.Data;
@Data
public class QueryDataReq {
String queryMode;
SchemaElement model;
Set<SchemaElement> metrics = new HashSet<>();
Set<SchemaElement> dimensions = new HashSet<>();
Set<QueryFilter> dimensionFilters = new HashSet<>();
Set<QueryFilter> metricFilters = new HashSet<>();
private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
private Set<Order> orders = new HashSet<>();
private DateConf dateInfo;
private Long limit;

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilters.java
View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class QueryFilters {
private List<QueryFilter> filters = new ArrayList<>();
private Map<String, Object> params = new HashMap<>();
}

1
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java
View File

@@ -5,7 +5,6 @@ import lombok.Data;
@Data
public class QueryReq {
private String queryText;
private Integer chatId;
private Long modelId = 0L;

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java
View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggRichConfigResp {

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java
View File

@@ -4,8 +4,10 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data

1
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java
View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java
View File

@@ -4,9 +4,10 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ChatDefaultRichConfigResp {

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java
View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailRichConfigResp {

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfoResp {
/**
* entity alias
* entity alias
*/
private List<String> names;

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestionResp.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
@Data
@AllArgsConstructor
public class RecommendQuestionResp {
private Long modelId;
private List<RecommendedQuestionReq> recommendedQuestions;
}

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class RecommendResp {
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
}

8
chat/core/pom.xml
View File

@@ -136,6 +136,14 @@
<artifactId>xk-time</artifactId>
<version>${xk.time.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>${mockito-inline.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

5
chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java
View File

@@ -3,12 +3,13 @@ package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.List;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
import lombok.ToString;
import java.util.List;
@Data
@ToString
public class ChatConfig {

1
chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java
View File

@@ -7,7 +7,6 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class FunctionCallInfoConfig {
@Value("${functionCall.url:}")
private String url;
}

43
chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java Normal file
View File

@@ -0,0 +1,43 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
@Configuration
@Data
@PropertySource("classpath:optimization.properties")
//@ComponentScan(basePackages = "com.tencent.supersonic.chat")
public class OptimizationConfig {
@Value("${one.detection.size}")
private Integer oneDetectionSize;
@Value("${one.detection.max.size}")
private Integer oneDetectionMaxSize;
@Value("${metric.dimension.min.threshold}")
private Double metricDimensionMinThresholdConfig;
@Value("${metric.dimension.threshold}")
private Double metricDimensionThresholdConfig;
@Value("${dimension.value.threshold}")
private Double dimensionValueThresholdConfig;
@Value("${function.bonus.threshold}")
private Double functionBonusThreshold;
@Value("${long.text.threshold}")
private Double longTextThreshold;
@Value("${short.text.threshold}")
private Double shortTextThreshold;
@Value("${query.text.length.threshold}")
private Integer queryTextLengthThreshold;
@Value("${candidate.threshold}")
private Double candidateThreshold;
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/DateFieldCorrector.java
View File

@@ -20,6 +20,7 @@ public class DateFieldCorrector extends BaseSemanticCorrector {
String currentDate = DSLDateHelper.getCurrentDate(correctionInfo.getParseInfo().getModelId());
sql = SqlParserUpdateHelper.addWhere(sql, DATE_FIELD, currentDate);
}
correctionInfo.setPreSql(correctionInfo.getSql());
correctionInfo.setSql(sql);
return correctionInfo;
}

6
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldCorrector.java
View File

@@ -9,9 +9,11 @@ public class FieldCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
String replaceFields = SqlParserUpdateHelper.replaceFields(correctionInfo.getSql(),
String preSql = correctionInfo.getSql();
correctionInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFields(preSql,
getFieldToBizName(correctionInfo.getParseInfo().getModelId()));
correctionInfo.setSql(replaceFields);
correctionInfo.setSql(sql);
return correctionInfo;
}
}

50
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldNameCorrector.java Normal file
View File

@@ -0,0 +1,50 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class FieldNameCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
Object context = correctionInfo.getParseInfo().getProperties().get(Constants.CONTEXT);
if (Objects.isNull(context)) {
return correctionInfo;
}
DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class);
if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) {
return correctionInfo;
}
LLMReq llmReq = dslParseResult.getLlmReq();
List<ElementValue> linking = llmReq.getLinking();
if (CollectionUtils.isEmpty(linking)) {
return correctionInfo;
}
Map<String, Set<String>> fieldValueToFieldNames = linking.stream().collect(
Collectors.groupingBy(ElementValue::getFieldValue,
Collectors.mapping(ElementValue::getFieldName, Collectors.toSet())));
String preSql = correctionInfo.getSql();
correctionInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFieldNameByValue(preSql, fieldValueToFieldNames);
correctionInfo.setSql(sql);
return correctionInfo;
}
}

79
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FieldValueCorrector.java
View File

@@ -1,18 +1,19 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import com.tencent.supersonic.knowledge.service.SchemaService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -20,29 +21,61 @@ public class FieldValueCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Long modelId = correctionInfo.getParseInfo().getModel().getId();
List<SchemaElement> dimensions = semanticSchema.getDimensions().stream()
.filter(schemaElement -> modelId.equals(schemaElement.getModel()))
.collect(Collectors.toList());
Object context = correctionInfo.getParseInfo().getProperties().get(Constants.CONTEXT);
if (Objects.isNull(context)) {
if (CollectionUtils.isEmpty(dimensions)) {
return correctionInfo;
}
DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class);
if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) {
return correctionInfo;
}
LLMReq llmReq = dslParseResult.getLlmReq();
List<ElementValue> linking = llmReq.getLinking();
if (CollectionUtils.isEmpty(linking)) {
return correctionInfo;
}
Map<String, Set<String>> fieldValueToFieldNames = linking.stream().collect(
Collectors.groupingBy(ElementValue::getFieldValue,
Collectors.mapping(ElementValue::getFieldName, Collectors.toSet())));
String sql = SqlParserUpdateHelper.replaceValueFields(correctionInfo.getSql(), fieldValueToFieldNames);
Map<String, Map<String, String>> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions);
String preSql = correctionInfo.getSql();
correctionInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceValue(preSql, aliasAndBizNameToTechName);
correctionInfo.setSql(sql);
return correctionInfo;
}
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(List<SchemaElement> dimensions) {
if (CollectionUtils.isEmpty(dimensions)) {
return new HashMap<>();
}
Map<String, Map<String, String>> result = new HashMap<>();
for (SchemaElement dimension : dimensions) {
if (Objects.isNull(dimension)
|| Strings.isEmpty(dimension.getBizName())
|| CollectionUtils.isEmpty(dimension.getSchemaValueMaps())) {
continue;
}
String bizName = dimension.getBizName();
Map<String, String> aliasAndBizNameToTechName = new HashMap<>();
for (SchemaValueMap valueMap : dimension.getSchemaValueMaps()) {
if (Objects.isNull(valueMap) || Strings.isEmpty(valueMap.getTechName())) {
continue;
}
if (Strings.isNotEmpty(valueMap.getBizName())) {
aliasAndBizNameToTechName.put(valueMap.getBizName(), valueMap.getTechName());
}
if (!CollectionUtils.isEmpty(valueMap.getAlias())) {
valueMap.getAlias().stream().forEach(alias -> {
if (Strings.isNotEmpty(alias)) {
aliasAndBizNameToTechName.put(alias, valueMap.getTechName());
}
});
}
}
if (!CollectionUtils.isEmpty(aliasAndBizNameToTechName)) {
result.put(bizName, aliasAndBizNameToTechName);
}
}
return result;
}
}

6
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/FunctionCorrector.java
View File

@@ -9,8 +9,10 @@ public class FunctionCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
String replaceFunction = SqlParserUpdateHelper.replaceFunction(correctionInfo.getSql());
correctionInfo.setSql(replaceFunction);
String preSql = correctionInfo.getSql();
correctionInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceFunction(preSql);
correctionInfo.setSql(sql);
return correctionInfo;
}
}

9
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/QueryFilterAppend.java
View File

@@ -20,14 +20,15 @@ public class QueryFilterAppend extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) throws JSQLParserException {
String queryFilter = getQueryFilter(correctionInfo.getQueryFilters());
String sql = correctionInfo.getSql();
String preSql = correctionInfo.getSql();
if (StringUtils.isNotEmpty(queryFilter)) {
log.info("add queryFilter to sql :{}", queryFilter);
log.info("add queryFilter to preSql :{}", queryFilter);
Expression expression = CCJSqlParserUtil.parseCondExpression(queryFilter);
sql = SqlParserUpdateHelper.addWhere(sql, expression);
String sql = SqlParserUpdateHelper.addWhere(preSql, expression);
correctionInfo.setPreSql(preSql);
correctionInfo.setSql(sql);
}
correctionInfo.setSql(sql);
return correctionInfo;
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrector.java
View File

@@ -15,24 +15,24 @@ public class SelectFieldAppendCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
String sql = correctionInfo.getSql();
if (SqlParserSelectHelper.hasAggregateFunction(sql)) {
String preSql = correctionInfo.getSql();
if (SqlParserSelectHelper.hasAggregateFunction(preSql)) {
return correctionInfo;
}
Set<String> selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(sql));
Set<String> whereFields = new HashSet<>(SqlParserSelectHelper.getWhereFields(sql));
Set<String> selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(preSql));
Set<String> whereFields = new HashSet<>(SqlParserSelectHelper.getWhereFields(preSql));
if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(whereFields)) {
return correctionInfo;
}
whereFields.addAll(SqlParserSelectHelper.getOrderByFields(sql));
whereFields.addAll(SqlParserSelectHelper.getOrderByFields(preSql));
whereFields.removeAll(selectFields);
whereFields.remove(TimeDimensionEnum.DAY.getName());
whereFields.remove(TimeDimensionEnum.WEEK.getName());
whereFields.remove(TimeDimensionEnum.MONTH.getName());
String replaceFields = SqlParserUpdateHelper.addFieldsToSelect(sql, new ArrayList<>(whereFields));
String replaceFields = SqlParserUpdateHelper.addFieldsToSelect(preSql, new ArrayList<>(whereFields));
correctionInfo.setPreSql(preSql);
correctionInfo.setSql(replaceFields);
return correctionInfo;
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableNameCorrector.java
View File

@@ -12,9 +12,10 @@ public class TableNameCorrector extends BaseSemanticCorrector {
@Override
public CorrectionInfo corrector(CorrectionInfo correctionInfo) {
Long modelId = correctionInfo.getParseInfo().getModelId();
String sqlOutput = correctionInfo.getSql();
String replaceTable = SqlParserUpdateHelper.replaceTable(sqlOutput, TABLE_PREFIX + modelId);
correctionInfo.setSql(replaceTable);
String preSql = correctionInfo.getSql();
correctionInfo.setPreSql(preSql);
String sql = SqlParserUpdateHelper.replaceTable(preSql, TABLE_PREFIX + modelId);
correctionInfo.setSql(sql);
return correctionInfo;
}

6
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
@@ -105,8 +106,9 @@ public class FuzzyNameMapper implements SchemaMapper {
private Double getThreshold(QueryContext queryContext, MapperHelper mapperHelper) {
Double metricDimensionThresholdConfig = mapperHelper.getMetricDimensionThresholdConfig();
Double metricDimensionMinThresholdConfig = mapperHelper.getMetricDimensionMinThresholdConfig();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
Double metricDimensionThresholdConfig = optimizationConfig.getMetricDimensionThresholdConfig();
Double metricDimensionMinThresholdConfig = optimizationConfig.getMetricDimensionMinThresholdConfig();
Map<Long, List<SchemaElementMatch>> modelElementMatches = queryContext.getMapInfo()
.getModelElementMatches();

23
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java
View File

@@ -9,13 +9,13 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.knowledge.utils.NatureHelper;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.dictionary.DictWordType;
import com.tencent.supersonic.knowledge.dictionary.MapResult;
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
import com.tencent.supersonic.knowledge.dictionary.builder.WordBuilderFactory;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import com.tencent.supersonic.knowledge.utils.NatureHelper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -25,6 +25,8 @@ import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@Slf4j
public class HanlpDictMapper implements SchemaMapper {
@@ -83,11 +85,14 @@ public class HanlpDictMapper implements SchemaMapper {
Long elementID = baseWordBuilder.getElementID(nature);
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
SchemaElement element = modelSchema.getElement(elementType, elementID);
if (Objects.isNull(element)) {
SchemaElement elementDb = modelSchema.getElement(elementType, elementID);
if (Objects.isNull(elementDb)) {
log.info("element is null, elementType:{},elementID:{}", elementType, elementID);
continue;
}
SchemaElement element = new SchemaElement();
BeanUtils.copyProperties(elementDb, element);
element.setAlias(getAlias(elementDb));
if (element.getType().equals(SchemaElementType.VALUE)) {
element.setName(mapResult.getName());
}
@@ -124,4 +129,16 @@ public class HanlpDictMapper implements SchemaMapper {
}
return matches;
}
public List<String> getAlias(SchemaElement element) {
if (!SchemaElementType.VALUE.equals(element.getType())) {
return element.getAlias();
}
if (CollectionUtils.isNotEmpty(element.getAlias()) && StringUtils.isNotEmpty(element.getName())) {
return element.getAlias().stream()
.filter(aliasItem -> aliasItem.contains(element.getName()))
.collect(Collectors.toList());
}
return element.getAlias();
}
}

23
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.mapper;
import com.hankcs.hanlp.algorithm.EditDistance;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.utils.NatureHelper;
@@ -13,7 +14,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
@@ -25,17 +26,8 @@ import org.springframework.stereotype.Service;
@Slf4j
public class MapperHelper {
@Value("${one.detection.size:8}")
private Integer oneDetectionSize;
@Value("${one.detection.max.size:20}")
private Integer oneDetectionMaxSize;
@Value("${metric.dimension.threshold:0.3}")
private Double metricDimensionThresholdConfig;
@Value("${metric.dimension.min.threshold:0.3}")
private Double metricDimensionMinThresholdConfig;
@Value("${dimension.value.threshold:0.5}")
private Double dimensionValueThresholdConfig;
@Autowired
private OptimizationConfig optimizationConfig;
public Integer getStepIndex(Map<Integer, Integer> regOffsetToLength, Integer index) {
Integer subRegLength = regOffsetToLength.get(index);
@@ -57,10 +49,11 @@ public class MapperHelper {
}
public double getThresholdMatch(List<String> natures) {
log.info("optimizationConfig:{}", optimizationConfig);
if (existDimensionValues(natures)) {
return dimensionValueThresholdConfig;
return optimizationConfig.getDimensionValueThresholdConfig();
}
return metricDimensionThresholdConfig;
return optimizationConfig.getMetricDimensionThresholdConfig();
}
/***
@@ -110,4 +103,4 @@ public class MapperHelper {
return detectModelIds;
}
}
}

10
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryMatchStrategy.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.mapper;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.knowledge.dictionary.MapResult;
import com.tencent.supersonic.knowledge.service.SearchService;
@@ -31,6 +32,9 @@ public class QueryMatchStrategy implements MatchStrategy {
@Autowired
private MapperHelper mapperHelper;
@Autowired
private OptimizationConfig optimizationConfig;
@Override
public Map<MatchText, List<MapResult>> match(QueryReq queryReq, List<Term> terms, Set<Long> detectModelIds) {
String text = queryReq.getQueryText();
@@ -111,7 +115,7 @@ public class QueryMatchStrategy implements MatchStrategy {
String detectSegment = text.substring(index, i);
// step1. pre search
Integer oneDetectionMaxSize = mapperHelper.getOneDetectionMaxSize();
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
LinkedHashSet<MapResult> mapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize, agentId,
detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
// step2. suffix search
@@ -153,7 +157,7 @@ public class QueryMatchStrategy implements MatchStrategy {
if (CollectionUtils.isNotEmpty(dimensionMetrics)) {
return dimensionMetrics;
} else {
return mapResults.stream().limit(mapperHelper.getOneDetectionSize()).collect(Collectors.toList());
return mapResults.stream().limit(optimizationConfig.getOneDetectionSize()).collect(Collectors.toList());
}
}
}
}

13
chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java
View File

@@ -4,7 +4,9 @@ package com.tencent.supersonic.chat.parser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
/**
@@ -15,10 +17,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SatisfactionChecker {
private static final double LONG_TEXT_THRESHOLD = 0.8;
private static final double SHORT_TEXT_THRESHOLD = 0.5;
private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10;
// check all the parse info in candidate
public static boolean check(QueryContext queryContext) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
@@ -35,11 +33,12 @@ public class SatisfactionChecker {
private static boolean checkThreshold(String queryText, SemanticParseInfo semanticParseInfo) {
int queryTextLength = queryText.replaceAll(" ", "").length();
double degree = semanticParseInfo.getScore() / queryTextLength;
if (queryTextLength > QUERY_TEXT_LENGTH_THRESHOLD) {
if (degree < LONG_TEXT_THRESHOLD) {
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (queryTextLength > optimizationConfig.getQueryTextLengthThreshold()) {
if (degree < optimizationConfig.getLongTextThreshold()) {
return false;
}
} else if (degree < SHORT_TEXT_THRESHOLD) {
} else if (degree < optimizationConfig.getShortTextThreshold()) {
return false;
}
log.info("queryMode:{}, degree:{}, parse info:{}",

16
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java
View File

@@ -93,19 +93,23 @@ public class LLMDslParser implements SemanticParser {
SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, dslTool, dslParseResult);
String correctorSql = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput());
CorrectionInfo correctionInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput());
llmResp.setCorrectorSql(correctorSql);
llmResp.setCorrectorSql(correctionInfo.getSql());
setFilter(correctorSql, modelId, parseInfo);
setFilter(correctionInfo, modelId, parseInfo);
} catch (Exception e) {
log.error("LLMDSLParser error", e);
}
}
public void setFilter(String correctorSql, Long modelId, SemanticParseInfo parseInfo) {
public void setFilter(CorrectionInfo correctionInfo, Long modelId, SemanticParseInfo parseInfo) {
String correctorSql = correctionInfo.getPreSql();
if (StringUtils.isEmpty(correctorSql)) {
correctorSql = correctionInfo.getSql();
}
List<FilterExpression> expressions = SqlParserSelectHelper.getFilterExpression(correctorSql);
if (CollectionUtils.isEmpty(expressions)) {
return;
@@ -200,7 +204,7 @@ public class LLMDslParser implements SemanticParser {
return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue());
}
private String getCorrectorSql(QueryContext queryCtx, SemanticParseInfo parseInfo, String sql) {
private CorrectionInfo getCorrectorSql(QueryContext queryCtx, SemanticParseInfo parseInfo, String sql) {
CorrectionInfo correctionInfo = CorrectionInfo.builder()
.queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql)
@@ -217,7 +221,7 @@ public class LLMDslParser implements SemanticParser {
log.error("sqlCorrection:{} execute error,correctionInfo:{}", dslCorrection, correctionInfo, e);
}
});
return correctionInfo.getSql();
return correctionInfo;
}
private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, DslTool dslTool,

118
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java Normal file
View File

@@ -0,0 +1,118 @@
package com.tencent.supersonic.chat.parser.plugin;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public abstract class PluginParser implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
if (!checkPreCondition(queryContext)) {
return;
}
PluginRecallResult pluginRecallResult = recallPlugin(queryContext);
if (pluginRecallResult == null) {
return;
}
buildQuery(queryContext, pluginRecallResult);
}
public abstract boolean checkPreCondition(QueryContext queryContext);
public abstract PluginRecallResult recallPlugin(QueryContext queryContext);
public void buildQuery(QueryContext queryContext, PluginRecallResult pluginRecallResult) {
Plugin plugin = pluginRecallResult.getPlugin();
for (Long modelId : pluginRecallResult.getModelIds()) {
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, queryContext.getRequest(),
queryContext.getMapInfo().getMatchedElements(modelId), pluginRecallResult.getDistance());
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
semanticParseInfo.setScore(pluginRecallResult.getScore());
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
if (plugin.isContainsAllModel()) {
break;
}
}
}
protected List<Plugin> getPluginList(QueryContext queryContext) {
return PluginManager.getPluginAgentCanSupport(queryContext.getRequest().getAgentId());
}
protected SemanticParseInfo buildSemanticParseInfo(Long modelId, Plugin plugin, QueryReq queryReq,
List<SchemaElementMatch> schemaElementMatches, double distance) {
if (modelId == null && !CollectionUtils.isEmpty(plugin.getModelList())) {
modelId = plugin.getModelList().get(0);
}
SchemaElement model = new SchemaElement();
model.setModel(modelId);
model.setId(modelId);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setElementMatches(schemaElementMatches);
semanticParseInfo.setModel(model);
Map<String, Object> properties = new HashMap<>();
PluginParseResult pluginParseResult = new PluginParseResult();
pluginParseResult.setPlugin(plugin);
pluginParseResult.setRequest(queryReq);
pluginParseResult.setDistance(distance);
properties.put(Constants.CONTEXT, pluginParseResult);
properties.put("type", "plugin");
properties.put("name", plugin.getName());
semanticParseInfo.setProperties(properties);
semanticParseInfo.setScore(distance);
fillSemanticParseInfo(semanticParseInfo);
setEntity(modelId, semanticParseInfo);
return semanticParseInfo;
}
private void setEntity(Long modelId, SemanticParseInfo semanticParseInfo) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (modelSchema != null && modelSchema.getEntity() != null) {
semanticParseInfo.setEntity(modelSchema.getEntity());
}
}
private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) {
List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
if (CollectionUtils.isEmpty(schemaElementMatches)) {
return;
}
schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
}
}

137
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java
View File

@@ -1,63 +1,54 @@
package com.tencent.supersonic.chat.parser.plugin.embedding;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashMap;
import java.util.Comparator;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
@Slf4j
public class EmbeddingBasedParser implements SemanticParser {
public class EmbeddingBasedParser extends PluginParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
public boolean checkPreCondition(QueryContext queryContext) {
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
if (StringUtils.isBlank(embeddingConfig.getUrl())) {
return;
return false;
}
log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext);
String text = queryContext.getRequest().getQueryText();
List<RecallRetrieval> embeddingRetrievals = recallResult(text);
choosePlugin(embeddingRetrievals, queryContext);
List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
for (SemanticQuery semanticQuery : semanticQueries) {
if (queryContext.getRequest().getQueryText().length() <= semanticQuery.getParseInfo().getScore()) {
return false;
}
}
return true;
}
private void choosePlugin(List<RecallRetrieval> embeddingRetrievals,
QueryContext queryContext) {
@Override
public PluginRecallResult recallPlugin(QueryContext queryContext) {
String text = queryContext.getRequest().getQueryText();
List<RecallRetrieval> embeddingRetrievals = embeddingRecall(text);
if (CollectionUtils.isEmpty(embeddingRetrievals)) {
return;
return null;
}
List<Plugin> plugins = getPluginList(queryContext);
Map<Long, Plugin> pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p));
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId()));
if (plugin == null || DslQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
if (plugin == null) {
continue;
}
Pair<Boolean, Set<Long>> pair = PluginManager.resolve(plugin, queryContext);
@@ -65,69 +56,19 @@ public class EmbeddingBasedParser implements SemanticParser {
if (pair.getLeft()) {
Set<Long> modelList = pair.getRight();
if (CollectionUtils.isEmpty(modelList)) {
return;
continue;
}
for (Long modelId : modelList) {
buildQuery(plugin, Double.parseDouble(embeddingRetrieval.getDistance()), modelId, queryContext,
queryContext.getMapInfo().getMatchedElements(modelId));
if (plugin.isContainsAllModel()) {
break;
}
}
return;
plugin.setParseMode(ParseMode.EMBEDDING_RECALL);
double distance = Double.parseDouble(embeddingRetrieval.getDistance());
double score = queryContext.getRequest().getQueryText().length() * (1 - distance);
return PluginRecallResult.builder()
.plugin(plugin).modelIds(modelList).score(score).distance(distance).build();
}
}
return null;
}
private void buildQuery(Plugin plugin, double distance, Long modelId,
QueryContext queryContext, List<SchemaElementMatch> schemaElementMatches) {
log.info("EmbeddingBasedParser Model: {} choose plugin: [{} {}]", modelId, plugin.getId(), plugin.getName());
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
plugin.setParseMode(ParseMode.EMBEDDING_RECALL);
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, plugin, queryContext.getRequest(),
schemaElementMatches, distance);
double score = queryContext.getRequest().getQueryText().length() * (1 - distance);
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
semanticParseInfo.setScore(score);
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
}
private SemanticParseInfo buildSemanticParseInfo(Long modelId, Plugin plugin, QueryReq queryReq,
List<SchemaElementMatch> schemaElementMatches, double distance) {
if (modelId == null && !CollectionUtils.isEmpty(plugin.getModelList())) {
modelId = plugin.getModelList().get(0);
}
SchemaElement model = new SchemaElement();
model.setModel(modelId);
model.setId(modelId);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setElementMatches(schemaElementMatches);
semanticParseInfo.setModel(model);
Map<String, Object> properties = new HashMap<>();
PluginParseResult pluginParseResult = new PluginParseResult();
pluginParseResult.setPlugin(plugin);
pluginParseResult.setRequest(queryReq);
pluginParseResult.setDistance(distance);
properties.put(Constants.CONTEXT, pluginParseResult);
properties.put("type", "plugin");
properties.put("name", plugin.getName());
semanticParseInfo.setProperties(properties);
semanticParseInfo.setScore(distance);
fillSemanticParseInfo(semanticParseInfo);
setEntity(modelId, semanticParseInfo);
return semanticParseInfo;
}
private void setEntity(Long modelId, SemanticParseInfo semanticParseInfo) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (modelSchema != null && modelSchema.getEntity() != null) {
semanticParseInfo.setEntity(modelSchema.getEntity());
}
}
public List<RecallRetrieval> recallResult(String embeddingText) {
public List<RecallRetrieval> embeddingRecall(String embeddingText) {
try {
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText);
@@ -144,26 +85,4 @@ public class EmbeddingBasedParser implements SemanticParser {
return Lists.newArrayList();
}
private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) {
List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
if (!CollectionUtils.isEmpty(schemaElementMatches)) {
schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
}
}
protected List<Plugin> getPluginList(QueryContext queryContext) {
return PluginManager.getPluginAgentCanSupport(queryContext.getRequest().getAgentId());
}
}

3
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingResp.java
View File

@@ -1,9 +1,10 @@
package com.tencent.supersonic.chat.parser.plugin.embedding;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class EmbeddingResp {

117
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java
View File

@@ -1,32 +1,23 @@
package com.tencent.supersonic.chat.parser.plugin.function;
import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.FunctionCallInfoConfig;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.Objects;
import java.util.Map;
import java.util.HashMap;
import java.util.stream.Collectors;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
@@ -42,85 +33,73 @@ import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
@Slf4j
public class FunctionBasedParser implements SemanticParser {
public class FunctionBasedParser extends PluginParser {
@Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) {
public boolean checkPreCondition(QueryContext queryContext) {
FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class);
PluginService pluginService = ContextUtils.getBean(PluginService.class);
String functionUrl = functionCallConfig.getUrl();
if (StringUtils.isBlank(functionUrl) || SatisfactionChecker.check(queryCtx)) {
if (StringUtils.isBlank(functionUrl) || SatisfactionChecker.check(queryContext)) {
log.info("functionUrl:{}, skip function parser, queryText:{}", functionUrl,
queryCtx.getRequest().getQueryText());
return;
queryContext.getRequest().getQueryText());
return false;
}
List<PluginParseConfig> functionDOList = getFunctionDO(queryCtx.getRequest().getModelId(), queryCtx);
if (CollectionUtils.isEmpty(functionDOList)) {
log.info("function call parser, plugin is empty, skip");
return;
}
FunctionResp functionResp = new FunctionResp();
if (functionDOList.size() == 1) {
functionResp.setToolSelection(functionDOList.iterator().next().getName());
} else {
FunctionReq functionReq = FunctionReq.builder()
.queryText(queryCtx.getRequest().getQueryText())
.pluginConfigs(functionDOList).build();
functionResp = requestFunction(functionUrl, functionReq);
return true;
}
@Override
public PluginRecallResult recallPlugin(QueryContext queryContext) {
PluginService pluginService = ContextUtils.getBean(PluginService.class);
FunctionResp functionResp = functionCall(queryContext);
if (skipFunction(functionResp)) {
return null;
}
log.info("requestFunction result:{}", functionResp.getToolSelection());
if (skipFunction(functionResp)) {
return;
}
PluginParseResult functionCallParseResult = new PluginParseResult();
String toolSelection = functionResp.getToolSelection();
Optional<Plugin> pluginOptional = pluginService.getPluginByName(toolSelection);
if (!pluginOptional.isPresent()) {
log.info("pluginOptional is not exist:{}, skip the parse", toolSelection);
return;
return null;
}
Plugin plugin = pluginOptional.get();
plugin.setParseMode(ParseMode.FUNCTION_CALL);
toolSelection = plugin.getType();
functionCallParseResult.setPlugin(plugin);
log.info("QueryManager PluginQueryModes:{}", QueryManager.getPluginQueryModes());
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(toolSelection);
ModelResolver modelResolver = ComponentFactory.getModelResolver();
log.info("plugin ModelList:{}", plugin.getModelList());
Pair<Boolean, Set<Long>> pluginResolveResult = PluginManager.resolve(plugin, queryCtx);
Long modelId = modelResolver.resolve(queryCtx, chatCtx, pluginResolveResult.getRight());
log.info("FunctionBasedParser modelId:{}", modelId);
if ((Objects.isNull(modelId) || modelId <= 0) && !plugin.isContainsAllModel()) {
log.info("Model is null, skip the parse, select tool: {}", toolSelection);
return;
Pair<Boolean, Set<Long>> pluginResolveResult = PluginManager.resolve(plugin, queryContext);
if (pluginResolveResult.getLeft()) {
Set<Long> modelList = pluginResolveResult.getRight();
if (CollectionUtils.isEmpty(modelList)) {
return null;
}
double score = queryContext.getRequest().getQueryText().length();
return PluginRecallResult.builder().plugin(plugin).modelIds(modelList).score(score).build();
}
if (!plugin.getModelList().contains(modelId) && !plugin.isContainsAllModel()) {
return;
return null;
}
public FunctionResp functionCall(QueryContext queryContext) {
List<PluginParseConfig> pluginToFunctionCall =
getPluginToFunctionCall(queryContext.getRequest().getModelId(), queryContext);
if (CollectionUtils.isEmpty(pluginToFunctionCall)) {
log.info("function call parser, plugin is empty, skip");
return null;
}
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
if (Objects.nonNull(modelId) && modelId > 0) {
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId));
FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class);
FunctionResp functionResp = new FunctionResp();
if (pluginToFunctionCall.size() == 1) {
functionResp.setToolSelection(pluginToFunctionCall.iterator().next().getName());
} else {
FunctionReq functionReq = FunctionReq.builder()
.queryText(queryContext.getRequest().getQueryText())
.pluginConfigs(pluginToFunctionCall).build();
functionResp = requestFunction(functionCallConfig.getUrl(), functionReq);
}
functionCallParseResult.setRequest(queryCtx.getRequest());
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, functionCallParseResult);
properties.put("type", "plugin");
properties.put("name", plugin.getName());
parseInfo.setProperties(properties);
parseInfo.setScore(queryCtx.getRequest().getQueryText().length());
parseInfo.setQueryMode(semanticQuery.getQueryMode());
SchemaElement model = new SchemaElement();
model.setModel(modelId);
model.setId(modelId);
parseInfo.setModel(model);
queryCtx.getCandidateQueries().add(semanticQuery);
return functionResp;
}
private boolean skipFunction(FunctionResp functionResp) {
return Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection());
}
private List<PluginParseConfig> getFunctionDO(Long modelId, QueryContext queryContext) {
private List<PluginParseConfig> getPluginToFunctionCall(Long modelId, QueryContext queryContext) {
log.info("user decide Model:{}", modelId);
List<Plugin> plugins = getPluginList(queryContext);
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
@@ -150,7 +129,7 @@ public class FunctionBasedParser implements SemanticParser {
return true;
}
}).map(o -> JsonUtil.toObject(o.getParseModeConfig(), PluginParseConfig.class)).collect(Collectors.toList());
log.info("getFunctionDO:{}", JsonUtil.toString(functionDOList));
log.info("PluginToFunctionCall: {}", JsonUtil.toString(functionDOList));
return functionDOList;
}
@@ -173,8 +152,4 @@ public class FunctionBasedParser implements SemanticParser {
}
return null;
}
protected List<Plugin> getPluginList(QueryContext queryContext) {
return PluginManager.getPluginAgentCanSupport(queryContext.getRequest().getAgentId());
}
}

3
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionReq.java
View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.chat.parser.plugin.function;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import java.util.List;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import lombok.Builder;
import lombok.Data;

1
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/ModelMatchResult.java
View File

@@ -4,7 +4,6 @@ import lombok.Data;
@Data
public class ModelMatchResult {
private Integer count = 0;
private double maxSimilarity;
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/Parameters.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.parser.plugin.function;
import lombok.Data;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class Parameters {

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java
View File

@@ -68,6 +68,7 @@ public class AggregateTypeParser implements SemanticParser {
Map<AggregateTypeEnum, Integer> aggregateCount = new HashMap<>(REGX_MAP.size());
Map<AggregateTypeEnum, String> aggregateWord = new HashMap<>(REGX_MAP.size());
for (Map.Entry<AggregateTypeEnum, Pattern> entry : REGX_MAP.entrySet()) {
Matcher matcher = entry.getValue().matcher(queryText);
int count = 0;
@@ -90,7 +91,6 @@ public class AggregateTypeParser implements SemanticParser {
@AllArgsConstructor
class AggregateConf {
public AggregateTypeEnum type;
public String detectWord;
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatConfigDO.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.persistence.dataobject;
import java.util.Date;
import lombok.Data;
import lombok.ToString;

1
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/ChatDO.java
View File

@@ -6,6 +6,7 @@ import lombok.Data;
public class ChatDO {
private long chatId;
private Integer agentId;
private String chatName;
private String createTime;
private String lastTime;

5
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/DimValueDO.java
View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.chat.persistence.dataobject;
import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import java.util.ArrayList;
import java.util.List;
import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import lombok.Data;
import lombok.ToString;

1
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.persistence.dataobject;
import java.util.Date;
public class PluginDO {
/**
*
*/

3
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDOExample.java
View File

@@ -5,7 +5,6 @@ import java.util.Date;
import java.util.List;
public class PluginDOExample {
/**
* s2_plugin
*/
@@ -149,7 +148,6 @@ public class PluginDOExample {
* s2_plugin null
*/
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
@@ -875,7 +873,6 @@ public class PluginDOExample {
* s2_plugin null
*/
public static class Criterion {
private String condition;
private Object value;

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatMapper.java
View File

@@ -10,7 +10,7 @@ public interface ChatMapper {
boolean createChat(ChatDO chatDO);
List<ChatDO> getAll(String creator);
List<ChatDO> getAll(String creator, Integer agentId);
Boolean updateChatName(Long chatId, String chatName, String lastTime, String creator);

3
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatQueryDOMapper.java
View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.persistence.mapper;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface ChatQueryDOMapper {

13
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java
View File

@@ -2,58 +2,67 @@ package com.tencent.supersonic.chat.persistence.mapper;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface PluginDOMapper {
/**
*
* @mbg.generated
*/
long countByExample(PluginDOExample example);
/**
*
* @mbg.generated
*/
int deleteByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int insert(PluginDO record);
/**
*
* @mbg.generated
*/
int insertSelective(PluginDO record);
/**
*
* @mbg.generated
*/
List<PluginDO> selectByExampleWithBLOBs(PluginDOExample example);
/**
*
* @mbg.generated
*/
List<PluginDO> selectByExample(PluginDOExample example);
/**
*
* @mbg.generated
*/
PluginDO selectByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeySelective(PluginDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeyWithBLOBs(PluginDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKey(PluginDO record);

3
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java
View File

@@ -1,9 +1,10 @@
package com.tencent.supersonic.chat.persistence.repository;
import com.tencent.supersonic.chat.config.ChatConfig;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.config.ChatConfig;
import java.util.List;
public interface ChatConfigRepository {

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatRepository.java
View File

@@ -8,7 +8,7 @@ public interface ChatRepository {
boolean createChat(ChatDO chatDO);
List<ChatDO> getAll(String creator);
List<ChatDO> getAll(String creator, Integer agentId);
Boolean updateChatName(Long chatId, String chatName, String lastTime, String creator);

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/PluginRepository.java
View File

@@ -2,10 +2,10 @@ package com.tencent.supersonic.chat.persistence.repository;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample;
import java.util.List;
public interface PluginRepository {
List<PluginDO> getPlugins();
List<PluginDO> fetchPluginDOs(String queryText, String type);

6
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatRepositoryImpl.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.persistence.repository.impl;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.persistence.dataobject.QueryDO;
import com.tencent.supersonic.chat.persistence.mapper.ChatMapper;
import com.tencent.supersonic.chat.persistence.repository.ChatRepository;
import com.tencent.supersonic.chat.persistence.mapper.ChatMapper;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
@@ -26,8 +26,8 @@ public class ChatRepositoryImpl implements ChatRepository {
}
@Override
public List<ChatDO> getAll(String creator) {
return chatMapper.getAll(creator);
public List<ChatDO> getAll(String creator, Integer agentId) {
return chatMapper.getAll(creator, agentId);
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java
View File

@@ -5,10 +5,10 @@ import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.common.pojo.RecordInfo;
import java.util.List;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
@Data
public class Plugin extends RecordInfo {

4
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java
View File

@@ -30,9 +30,7 @@ import java.util.HashSet;
import java.util.HashMap;
import java.util.Objects;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -52,8 +50,6 @@ import org.springframework.web.util.UriComponentsBuilder;
@Component
public class PluginManager {
private static Map<String, Plugin> internalPluginMap = new ConcurrentHashMap<>();
private EmbeddingConfig embeddingConfig;
private RestTemplate restTemplate;

23
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginRecallResult.java Normal file
View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.chat.plugin;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Set;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PluginRecallResult {
private Plugin plugin;
private Set<Long> modelIds;
private double score;
private double distance;
}

9
chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java
View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import java.util.List;
@@ -12,18 +13,18 @@ import java.util.OptionalDouble;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class HeuristicQuerySelector implements QuerySelector {
private static final double CANDIDATE_THRESHOLD = 0.2;
@Override
public List<SemanticQuery> select(List<SemanticQuery> candidateQueries, QueryReq queryReq) {
List<SemanticQuery> selectedQueries = new ArrayList<>();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
Double candidateThreshold = optimizationConfig.getCandidateThreshold();
if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) {
selectedQueries.addAll(candidateQueries);
} else {
@@ -35,7 +36,7 @@ public class HeuristicQuerySelector implements QuerySelector {
candidateQueries.stream().forEach(query -> {
SemanticParseInfo parseInfo = query.getParseInfo();
if (!checkFullyInherited(query)
&& (maxScore - parseInfo.getScore()) / maxScore <= CANDIDATE_THRESHOLD
&& (maxScore - parseInfo.getScore()) / maxScore <= candidateThreshold
&& checkSatisfyOtherRules(query, candidateQueries)) {
selectedQueries.add(query);
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -55,7 +56,6 @@ public class QueryManager {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
}
public static boolean containsRuleQuery(String queryMode) {
if (queryMode == null) {
return false;

1
chat/core/src/main/java/com/tencent/supersonic/chat/query/metricInterpret/LLmAnswerResp.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.chat.query.metricinterpret;
import lombok.Data;
@Data

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class WebBase {

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBaseResult.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class WebBaseResult {

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.query.plugin.webpage;
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class WebPageResponse {

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -1,3 +1,4 @@
package com.tencent.supersonic.chat.query.rule;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -291,6 +292,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
}
protected QueryStructReq convertQueryStruct() {
return QueryReqBuilder.buildStructReq(parseInfo);
}

15
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java
View File

@@ -1,9 +1,5 @@
package com.tencent.supersonic.chat.query.rule.entity;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
@@ -15,12 +11,17 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
@Slf4j
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
@@ -34,7 +35,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -40,7 +40,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.chat.query.rule.metric;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,11 +13,12 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.springframework.stereotype.Component;
@Component
public class MetricTopNQuery extends MetricSemanticQuery {
@@ -35,7 +36,7 @@ public class MetricTopNQuery extends MetricSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
QueryContext queryCtx) {
Matcher matcher = INTENT_PATTERN.matcher(queryCtx.getRequest().getQueryText());
if (matcher.matches()) {
return super.match(candidateElementMatches, queryCtx);

8
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java
View File

@@ -30,14 +30,16 @@ public class ChatController {
@PostMapping("/save")
public Boolean save(@RequestParam(value = "chatName") String chatName,
@RequestParam(value = "agentId", required = false) Integer agentId,
HttpServletRequest request, HttpServletResponse response) {
return chatService.addChat(UserHolder.findUser(request, response), chatName);
return chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
}
@GetMapping("/getAll")
public List<ChatDO> getAllConversions(HttpServletRequest request, HttpServletResponse response) {
public List<ChatDO> getAllConversions(@RequestParam(value = "agentId", required = false) Integer agentId,
HttpServletRequest request, HttpServletResponse response) {
String userName = UserHolder.findUser(request, response).getName();
return chatService.getAll(userName);
return chatService.getAll(userName, agentId);
}
@PostMapping("/delete")

16
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.rest;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
@@ -33,7 +34,7 @@ public class ChatQueryController {
@PostMapping("search")
public Object search(@RequestBody QueryReq queryCtx, HttpServletRequest request,
HttpServletResponse response) {
HttpServletResponse response) {
queryCtx.setUser(UserHolder.findUser(request, response));
return searchService.search(queryCtx);
}
@@ -54,7 +55,7 @@ public class ChatQueryController {
@PostMapping("execute")
public Object execute(@RequestBody ExecuteQueryReq queryCtx,
HttpServletRequest request, HttpServletResponse response)
HttpServletRequest request, HttpServletResponse response)
throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.performExecution(queryCtx);
@@ -62,16 +63,23 @@ public class ChatQueryController {
@PostMapping("queryContext")
public Object queryContext(@RequestBody QueryReq queryCtx, HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response) throws Exception {
queryCtx.setUser(UserHolder.findUser(request, response));
return queryService.queryContext(queryCtx);
}
@PostMapping("queryData")
public Object queryData(@RequestBody QueryDataReq queryData,
HttpServletRequest request, HttpServletResponse response)
HttpServletRequest request, HttpServletResponse response)
throws Exception {
return queryService.executeDirectQuery(queryData, UserHolder.findUser(request, response));
}
@PostMapping("queryDimensionValue")
public Object queryDimensionValue(@RequestBody DimensionValueReq dimensionValueReq,
HttpServletRequest request, HttpServletResponse response)
throws Exception {
return queryService.queryDimensionValue(dimensionValueReq, UserHolder.findUser(request, response));
}
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/rest/DictionaryController.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;

4
chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java
View File

@@ -30,9 +30,9 @@ public interface ChatService {
public void switchContext(ChatContext chatCtx);
public Boolean addChat(User user, String chatName);
public Boolean addChat(User user, String chatName, Integer agentId);
public List<ChatDO> getAll(String userName);
public List<ChatDO> getAll(String userName, Integer agentId);
public boolean updateChatName(Long chatId, String chatName, String userName);

1
chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java
View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import java.util.List;
public interface ConfigService {

2
chat/core/src/main/java/com/tencent/supersonic/chat/service/DictionaryService.java
View File

@@ -5,10 +5,10 @@ import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import java.util.List;
public interface DictionaryService {
Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);

3
chat/core/src/main/java/com/tencent/supersonic/chat/service/PluginService.java
View File

@@ -2,8 +2,9 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq;
import java.util.List;
import java.util.Optional;

4
chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java
View File

@@ -2,11 +2,12 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
import org.apache.calcite.sql.parser.SqlParseException;
/***
@@ -24,4 +25,5 @@ public interface QueryService {
QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException;
Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception;
}

1
chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java
View File

@@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import java.util.List;
/***

8
chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java
View File

@@ -26,9 +26,9 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.DataInfo;
import com.tencent.supersonic.chat.api.pojo.response.ModelInfo;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
import com.tencent.supersonic.chat.api.pojo.response.ModelInfo;
import com.tencent.supersonic.chat.config.AggregatorConfig;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
@@ -332,7 +332,7 @@ public class SemanticService {
}
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp result) {
QueryResultWithSchemaResp result) {
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) {
return new AggregateInfo();
}
@@ -384,7 +384,7 @@ public class SemanticService {
}
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) {
AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) {
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
@@ -432,7 +432,7 @@ public class SemanticService {
}
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp results) {
QueryResultWithSchemaResp results) {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp = results.getResultList().stream()
.map(r -> r.get(dateField).toString())

25
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -82,21 +82,22 @@ public class ChatServiceImpl implements ChatService {
@Override
public Boolean addChat(User user, String chatName) {
ChatDO intelligentConversionDO = new ChatDO();
intelligentConversionDO.setChatName(chatName);
intelligentConversionDO.setCreator(user.getName());
intelligentConversionDO.setCreateTime(getCurrentTime());
intelligentConversionDO.setIsDelete(0);
intelligentConversionDO.setLastTime(getCurrentTime());
intelligentConversionDO.setLastQuestion("Hello, welcome to using supersonic");
intelligentConversionDO.setIsTop(0);
return chatRepository.createChat(intelligentConversionDO);
public Boolean addChat(User user, String chatName, Integer agentId) {
ChatDO chatDO = new ChatDO();
chatDO.setChatName(chatName);
chatDO.setCreator(user.getName());
chatDO.setCreateTime(getCurrentTime());
chatDO.setIsDelete(0);
chatDO.setLastTime(getCurrentTime());
chatDO.setLastQuestion("Hello, welcome to using supersonic");
chatDO.setIsTop(0);
chatDO.setAgentId(agentId);
return chatRepository.createChat(chatDO);
}
@Override
public List<ChatDO> getAll(String userName) {
return chatRepository.getAll(userName);
public List<ChatDO> getAll(String userName, Integer agentId) {
return chatRepository.getAll(userName, agentId);
}
@Override

12
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java
View File

@@ -9,15 +9,15 @@ import com.tencent.supersonic.chat.utils.DictMetaHelper;
import com.tencent.supersonic.chat.utils.DictQueryHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -41,9 +41,9 @@ public class DictionaryServiceImpl implements DictionaryService {
private String dimValue = "DimValue_%d_%d";
public DictionaryServiceImpl(DictMetaHelper metaUtils,
DictQueryHelper dictQueryHelper,
FileHandler fileHandler,
DictRepository dictRepository) {
DictQueryHelper dictQueryHelper,
FileHandler fileHandler,
DictRepository dictRepository) {
this.metaUtils = metaUtils;
this.dictQueryHelper = dictQueryHelper;
this.fileHandler = fileHandler;

62
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
@@ -26,13 +27,23 @@ import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.service.StatisticsService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
import java.util.Comparator;
import java.util.Objects;
import java.util.stream.Collectors;
//import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
//import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.springframework.beans.BeanUtils;
@@ -239,5 +250,52 @@ public class QueryServiceImpl implements QueryService {
return semanticQuery.execute(user);
}
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
com.tencent.supersonic.semantic.query.service.QueryService queryService =
ContextUtils.getBean(com.tencent.supersonic.semantic.query.service.QueryService.class);
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateConf.DateMode.RECENT);
dateConf.setUnit(1);
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
// List<Aggregator> aggregators = new ArrayList<>();
// Aggregator aggregator = new Aggregator(dimensionValueReq.getQueryFilter().getBizName(),
// AggOperatorEnum.DISTINCT);
// aggregators.add(aggregator);
// queryStructReq.setAggregators(aggregators);
queryStructReq.setModelId(dimensionValueReq.getModelId());
queryStructReq.setNativeQuery(true);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);
if (Objects.isNull(dimensionValueReq.getValue())) {
List<Filter> dimensionFilters = new ArrayList<>();
Filter dimensionFilter = new Filter();
dimensionFilter.setOperator(FilterOperatorEnum.LIKE);
dimensionFilter.setRelation(Filter.Relation.FILTER);
dimensionFilter.setBizName(dimensionValueReq.getBizName());
dimensionFilter.setValue(dimensionValueReq.getValue());
dimensionFilters.add(dimensionFilter);
queryStructReq.setDimensionFilters(dimensionFilters);
}
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStructWithAuth(queryStructReq, user);
Set<String> dimensionValues = new HashSet<>();
queryResultWithSchemaResp.getResultList().removeIf(o -> {
if (dimensionValues.contains(o.get(dimensionValueReq.getBizName()))) {
return true;
} else {
dimensionValues.add(o.get(dimensionValueReq.getBizName()).toString());
return false;
}
});
return queryResultWithSchemaResp;
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java
View File

@@ -4,10 +4,10 @@ import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import java.util.concurrent.TimeUnit;
public class CacheUtils {
private static final Cache<String, Object> cache = Caffeine.newBuilder()
.expireAfterWrite(1200, TimeUnit.SECONDS)
.expireAfterAccess(1200, TimeUnit.SECONDS)

16
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java
View File

@@ -9,16 +9,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -26,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.StringJoiner;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
@@ -46,7 +48,7 @@ public class DictQueryHelper {
private Long dimMaxLimit;
public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
User user) {
User user) {
List<String> data = new ArrayList<>();
QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
try {
@@ -89,7 +91,7 @@ public class DictQueryHelper {
}
private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
String metricName) {
String metricName) {
List<String> data = new ArrayList<>();
if (CollectionUtils.isEmpty(resultList)) {
return data;

19
chat/core/src/main/resources/mapper/ChatConfigMapper.xml
View File

@@ -6,7 +6,7 @@
<mapper namespace="com.tencent.supersonic.chat.persistence.mapper.ChatConfigMapper">
<resultMap id="chaConfigDO"
type="com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO">
type="com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO">
<id column="id" property="id"/>
<result column="model_id" property="modelId"/>
<result column="chat_detail_config" property="chatDetailConfig"/>
@@ -22,11 +22,14 @@
<insert id="addConfig"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO"
useGeneratedKeys="true" keyProperty="id">
insert into s2_chat_config
(model_id, `chat_detail_config`, chat_agg_config, recommended_questions, status, created_by,
updated_by, created_at, updated_at)
values (#{modelId}, #{chatDetailConfig}, #{chatAggConfig}, #{recommendedQuestions},
#{status}, #{createdBy}, #{updatedBy}, #{createdAt}, #{updatedAt})
insert into s2_chat_config
(
model_id, `chat_detail_config`, chat_agg_config, recommended_questions, status, created_by, updated_by, created_at, updated_at
)
values
(
#{modelId}, #{chatDetailConfig}, #{chatAggConfig}, #{recommendedQuestions}, #{status}, #{createdBy}, #{updatedBy}, #{createdAt}, #{updatedAt}
)
</insert>
@@ -82,9 +85,9 @@
select *
from s2_chat_config
where model_id = #{modelId}
and status != 3
and status != 3
order by updated_at desc
limit 1
limit 1
</select>
</mapper>

18
chat/core/src/main/resources/mapper/ChatContextMapper.xml
View File

@@ -6,7 +6,7 @@
<mapper namespace="com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper">
<resultMap id="ChatContextDO"
type="com.tencent.supersonic.chat.persistence.dataobject.ChatContextDO">
type="com.tencent.supersonic.chat.persistence.dataobject.ChatContextDO">
<id column="chat_id" property="chatId"/>
<result column="modified_at" property="modifiedAt"/>
<result column="user" property="user"/>
@@ -15,22 +15,16 @@
<!--<result column="ext_data" property="extData"/>-->
</resultMap>
<select id="getContextByChatId" resultMap="ChatContextDO">
<select id="getContextByChatId" resultMap="ChatContextDO">
select *
from s2_chat_context
where chat_id = #{chatId} limit 1
from s2_chat_context where chat_id=#{chatId} limit 1
</select>
<insert id="addContext"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatContextDO">
insert into s2_chat_context (chat_id, user, query_text, semantic_parse)
values (#{chatId}, #{user}, #{queryText}, #{semanticParse})
<insert id="addContext" parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatContextDO" >
insert into s2_chat_context (chat_id,user,query_text,semantic_parse) values (#{chatId}, #{user},#{queryText}, #{semanticParse})
</insert>
<update id="updateContext">
update s2_chat_context
set query_text=#{queryText},
semantic_parse=#{semanticParse}
where chat_id = #{chatId}
update s2_chat_context set query_text=#{queryText},semantic_parse=#{semanticParse} where chat_id=#{chatId}
</update>
</mapper>

41
chat/core/src/main/resources/mapper/ChatMapper.xml
View File

@@ -5,9 +5,9 @@
<mapper namespace="com.tencent.supersonic.chat.persistence.mapper.ChatMapper">
<resultMap id="IntelligentConversion"
type="com.tencent.supersonic.chat.persistence.dataobject.ChatDO">
<resultMap id="IntelligentConversion" type="com.tencent.supersonic.chat.persistence.dataobject.ChatDO">
<id column="chat_id" property="chatId"/>
<result column="agent_id" property="agentId"/>
<result column="chat_name" property="chatName"/>
<result column="create_time" property="createTime"/>
<result column="last_time" property="lastTime"/>
@@ -22,6 +22,9 @@
from s2_chat
where creator = #{creator}
and is_delete = 0
<if test="agentId != null">
and agent_id = #{agentId}
</if>
order by is_top desc, last_time desc
</select>
@@ -42,16 +45,14 @@
<update id="updateLastQuestion">
update s2_chat
set last_question = #{lastQuestion},
last_time = #{lastTime}
last_time = #{lastTime}
where chat_id = #{chatId}
</update>
<insert id="createChat"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatDO">
<insert id="createChat" parameterType="com.tencent.supersonic.chat.persistence.dataobject.ChatDO">
insert into s2_chat
(chat_name, create_time, last_time, creator, last_question, is_delete, is_top)
values (#{chatName}, #{createTime}, #{lastTime}, #{creator}, #{lastQuestion}, #{isDelete},
#{isTop})
(agent_id, chat_name, create_time, last_time, creator, last_question, is_delete, is_top)
values (#{agentId}, #{chatName}, #{createTime}, #{lastTime}, #{creator}, #{lastQuestion}, #{isDelete}, #{isTop})
</insert>
<update id="deleteChat">
@@ -62,8 +63,7 @@
</update>
<resultMap id="IntelligentQuery"
type="com.tencent.supersonic.chat.persistence.dataobject.QueryDO">
<resultMap id="IntelligentQuery" type="com.tencent.supersonic.chat.persistence.dataobject.QueryDO">
<id column="id" property="id"/>
<result column="question_id" property="questionId"/>
<result column="create_time" property="createTime"/>
@@ -87,29 +87,22 @@
<result column="chat_id" property="chatId"/>
</resultMap>
<update id="updateFeedback"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.QueryDO">
<update id="updateFeedback" parameterType="com.tencent.supersonic.chat.persistence.dataobject.QueryDO">
update s2_chat_query
set score=#{score},
feedback=#{feedback}
where question_id = #{id}
</update>
<insert id="createQuery"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.QueryDO"
<insert id="createQuery" parameterType="com.tencent.supersonic.chat.persistence.dataobject.QueryDO"
useGeneratedKeys="true" keyProperty="id">
insert into s2_chat_query
(question_id, create_time, user_name, question, query_result, time, state, data_content,
name, query_type,
is_deleted, module, chat_id, aggregator, top_num, start_time, end_time, query_sql, columns,
main_entity, score,
(question_id, create_time, user_name, question, query_result, time, state, data_content, name, query_type,
is_deleted, module, chat_id, aggregator, top_num, start_time, end_time, query_sql, columns, main_entity, score,
feedback)
values (#{questionId}, #{createTime}, #{userName}, #{question}, #{queryResults}, #{time},
#{state},
#{dataContent}, #{name}, #{queryType}, #{isDeleted}, #{module}, #{chatId},
#{aggregator}, #{topNum},
#{startTime}, #{endTime}, #{querySql}, #{QueryColumn}, #{EntityInfo}, #{score},
#{feedback})
values (#{questionId}, #{createTime}, #{userName}, #{question}, #{queryResults}, #{time}, #{state},
#{dataContent}, #{name}, #{queryType}, #{isDeleted}, #{module}, #{chatId}, #{aggregator}, #{topNum},
#{startTime}, #{endTime}, #{querySql}, #{QueryColumn}, #{EntityInfo}, #{score}, #{feedback})
</insert>

563
chat/core/src/main/resources/mapper/PluginDOMapper.xml
View File

@@ -1,297 +1,276 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.chat.persistence.mapper.PluginDOMapper">
<resultMap id="BaseResultMap"
type="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
<id column="id" jdbcType="BIGINT" property="id"/>
<result column="type" jdbcType="VARCHAR" property="type"/>
<result column="model" jdbcType="VARCHAR" property="model"/>
<result column="pattern" jdbcType="VARCHAR" property="pattern"/>
<result column="parse_mode" jdbcType="VARCHAR" property="parseMode"/>
<result column="name" jdbcType="VARCHAR" property="name"/>
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt"/>
<result column="created_by" jdbcType="VARCHAR" property="createdBy"/>
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt"/>
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy"/>
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs"
type="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
<result column="parse_mode_config" jdbcType="LONGVARCHAR" property="parseModeConfig"/>
<result column="config" jdbcType="LONGVARCHAR" property="config"/>
<result column="comment" jdbcType="LONGVARCHAR" property="comment"/>
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and
#{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem"
open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
<resultMap id="BaseResultMap" type="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
<id column="id" jdbcType="BIGINT" property="id" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="model" jdbcType="VARCHAR" property="model" />
<result column="pattern" jdbcType="VARCHAR" property="pattern" />
<result column="parse_mode" jdbcType="VARCHAR" property="parseMode" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
<result column="parse_mode_config" jdbcType="LONGVARCHAR" property="parseModeConfig" />
<result column="config" jdbcType="LONGVARCHAR" property="config" />
<result column="comment" jdbcType="LONGVARCHAR" property="comment" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id
, type, model, pattern, parse_mode, name, created_at, created_by, updated_at,
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, type, model, pattern, parse_mode, name, created_at, created_by, updated_at,
updated_by
</sql>
<sql id="Blob_Column_List">
parse_mode_config
, config, comment
</sql>
<select id="selectByExampleWithBLOBs"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample"
resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List"/>
,
<include refid="Blob_Column_List"/>
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample"
resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List"/>
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="limitStart != null and limitStart>=0">
limit #{limitStart} , #{limitEnd}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List"/>
,
<include refid="Blob_Column_List"/>
from s2_plugin
where id = #{id,jdbcType=BIGINT}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete
from s2_plugin
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
insert into s2_plugin (id, type, model,
pattern, parse_mode, name,
created_at, created_by, updated_at,
updated_by, parse_mode_config, config,
comment)
values (#{id,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{model,jdbcType=VARCHAR},
#{pattern,jdbcType=VARCHAR}, #{parseMode,jdbcType=VARCHAR},
#{name,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR},
#{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{parseModeConfig,jdbcType=LONGVARCHAR},
#{config,jdbcType=LONGVARCHAR},
#{comment,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
insert into s2_plugin
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="type != null">
type,
</if>
<if test="model != null">
model,
</if>
<if test="pattern != null">
pattern,
</if>
<if test="parseMode != null">
parse_mode,
</if>
<if test="name != null">
name,
</if>
<if test="createdAt != null">
created_at,
</if>
<if test="createdBy != null">
created_by,
</if>
<if test="updatedAt != null">
updated_at,
</if>
<if test="updatedBy != null">
updated_by,
</if>
<if test="parseModeConfig != null">
parse_mode_config,
</if>
<if test="config != null">
config,
</if>
<if test="comment != null">
comment,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=BIGINT},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
#{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
#{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
#{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
#{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
#{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
#{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
#{comment,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample"
resultType="java.lang.Long">
select count(*) from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause"/>
</if>
</select>
<update id="updateByPrimaryKeySelective"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
<set>
<if test="type != null">
type = #{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
model = #{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
pattern = #{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
parse_mode = #{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
name = #{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
created_by = #{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
comment = #{comment,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKeyWithBLOBs"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
config = #{config,jdbcType=LONGVARCHAR},
comment = #{comment,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey"
parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</sql>
<sql id="Blob_Column_List">
parse_mode_config, config, comment
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="limitStart != null and limitStart>=0">
limit #{limitStart} , #{limitEnd}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_plugin
where id = #{id,jdbcType=BIGINT}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete from s2_plugin
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
insert into s2_plugin (id, type, model,
pattern, parse_mode, name,
created_at, created_by, updated_at,
updated_by, parse_mode_config, config,
comment)
values (#{id,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{model,jdbcType=VARCHAR},
#{pattern,jdbcType=VARCHAR}, #{parseMode,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{parseModeConfig,jdbcType=LONGVARCHAR}, #{config,jdbcType=LONGVARCHAR},
#{comment,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
insert into s2_plugin
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="type != null">
type,
</if>
<if test="model != null">
model,
</if>
<if test="pattern != null">
pattern,
</if>
<if test="parseMode != null">
parse_mode,
</if>
<if test="name != null">
name,
</if>
<if test="createdAt != null">
created_at,
</if>
<if test="createdBy != null">
created_by,
</if>
<if test="updatedAt != null">
updated_at,
</if>
<if test="updatedBy != null">
updated_by,
</if>
<if test="parseModeConfig != null">
parse_mode_config,
</if>
<if test="config != null">
config,
</if>
<if test="comment != null">
comment,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=BIGINT},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
#{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
#{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
#{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
#{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
#{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
#{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
#{comment,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample" resultType="java.lang.Long">
select count(*) from s2_plugin
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
<set>
<if test="type != null">
type = #{type,jdbcType=VARCHAR},
</if>
<if test="model != null">
model = #{model,jdbcType=VARCHAR},
</if>
<if test="pattern != null">
pattern = #{pattern,jdbcType=VARCHAR},
</if>
<if test="parseMode != null">
parse_mode = #{parseMode,jdbcType=VARCHAR},
</if>
<if test="name != null">
name = #{name,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
created_by = #{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="parseModeConfig != null">
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
<if test="comment != null">
comment = #{comment,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR},
config = #{config,jdbcType=LONGVARCHAR},
comment = #{comment,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.chat.persistence.dataobject.PluginDO">
update s2_plugin
set type = #{type,jdbcType=VARCHAR},
model = #{model,jdbcType=VARCHAR},
pattern = #{pattern,jdbcType=VARCHAR},
parse_mode = #{parseMode,jdbcType=VARCHAR},
name = #{name,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>

26
chat/core/src/main/resources/sql.ddl/chat.sql
View File

@@ -2,10 +2,10 @@ CREATE TABLE `chat_context`
(
`chat_id` bigint(20) NOT NULL COMMENT 'context chat id',
`modified_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'row modify time',
`user` varchar(64) DEFAULT NULL COMMENT 'row modify user',
`query_text` text DEFAULT NULL COMMENT 'query text',
`semantic_parse` text DEFAULT NULL COMMENT 'parse data',
`ext_data` text DEFAULT NULL COMMENT 'extend data',
`user` varchar(64) DEFAULT NULL COMMENT 'row modify user',
`query_text` text DEFAULT NULL COMMENT 'query text',
`semantic_parse` text DEFAULT NULL COMMENT 'parse data',
`ext_data` text DEFAULT NULL COMMENT 'extend data',
PRIMARY KEY (`chat_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -40,7 +40,7 @@ CREATE TABLE `chat_query`
`is_deleted` int(1) DEFAULT NULL,
`module` varchar(30) DEFAULT NULL,
`entity` mediumtext COMMENT '',
`chat_id` bigint(8) DEFAULT NULL COMMENT 'chat id',
`chat_id` bigint(8) DEFAULT NULL COMMENT 'chat id',
`recommend` text,
`aggregator` varchar(20) DEFAULT 'trend',
`top_num` int(3) DEFAULT NULL,
@@ -59,18 +59,4 @@ CREATE TABLE `chat_query`
KEY `common` (`question_id`),
KEY `common1` (`user_name`),
KEY `common2` (`chat_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `chat`
(
`chat_id` bigint(8) NOT NULL AUTO_INCREMENT,
`chat_name` varchar(100) DEFAULT NULL,
`create_time` datetime DEFAULT NULL,
`last_time` datetime DEFAULT NULL,
`creator` varchar(30) DEFAULT NULL,
`last_question` varchar(200) DEFAULT NULL,
`is_delete` int(2) DEFAULT '0' COMMENT 'is deleted',
`is_top` int(2) DEFAULT '0' COMMENT 'is top',
PRIMARY KEY (`chat_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

3
chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/DateFieldCorrectorTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/corrector/DateFieldCorrectorTest.java
View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.chat.query.llm.dsl.corrector;
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.corrector.DateFieldCorrector;
import org.junit.Assert;
import org.junit.jupiter.api.Test;

7
chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/FieldValueCorrectorTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldNameCorrectorTest.java
View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.chat.query.llm.dsl.corrector;
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.corrector.FieldValueCorrector;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
@@ -14,12 +13,12 @@ import java.util.Map;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
class FieldValueCorrectorTest {
class FieldNameCorrectorTest {
@Test
void rewriter() {
FieldValueCorrector corrector = new FieldValueCorrector();
FieldNameCorrector corrector = new FieldNameCorrector();
CorrectionInfo correctionInfo = CorrectionInfo.builder()
.sql("select 歌曲名 from 歌曲库 where 专辑照片 = '七里香' and 专辑名 = '流行' and 数据日期 = '2023-08-19'")
.build();

71
chat/core/src/test/java/com/tencent/supersonic/chat/corrector/FieldValueCorrectorTest.java Normal file
View File

@@ -0,0 +1,71 @@
package com.tencent.supersonic.chat.corrector;
import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
class FieldValueCorrectorTest {
@Test
void corrector() {
MockedStatic<ContextUtils> mockContextUtils = Mockito.mockStatic(ContextUtils.class);
SchemaService mockSchemaService = Mockito.mock(SchemaService.class);
SemanticSchema mockSemanticSchema = Mockito.mock(SemanticSchema.class);
List<SchemaElement> dimensions = new ArrayList<>();
List<SchemaValueMap> schemaValueMaps = new ArrayList<>();
SchemaValueMap value1 = new SchemaValueMap();
value1.setBizName("杰伦");
value1.setTechName("周杰伦");
value1.setAlias(Arrays.asList("周杰倫", "Jay Chou", "周董", "周先生"));
schemaValueMaps.add(value1);
SchemaElement schemaElement = SchemaElement.builder()
.bizName("singer_name")
.name("歌手名")
.model(2L)
.schemaValueMaps(schemaValueMaps)
.build();
dimensions.add(schemaElement);
when(mockSemanticSchema.getDimensions()).thenReturn(dimensions);
when(mockSchemaService.getSemanticSchema()).thenReturn(mockSemanticSchema);
mockContextUtils.when(() -> ContextUtils.getBean(SchemaService.class)).thenReturn(mockSchemaService);
SemanticParseInfo parseInfo = new SemanticParseInfo();
SchemaElement model = new SchemaElement();
model.setId(2L);
parseInfo.setModel(model);
CorrectionInfo correctionInfo = CorrectionInfo.builder()
.sql("select count(song_name) from 歌曲库 where singer_name = '周先生'")
.parseInfo(parseInfo)
.build();
FieldValueCorrector corrector = new FieldValueCorrector();
CorrectionInfo info = corrector.corrector(correctionInfo);
Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", info.getSql());
correctionInfo.setSql("select count(song_name) from 歌曲库 where singer_name = '杰伦'");
info = corrector.corrector(correctionInfo);
Assert.assertEquals("SELECT count(song_name) FROM 歌曲库 WHERE singer_name = '周杰伦'", info.getSql());
}
}

3
chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/SelectFieldAppendCorrectorTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/corrector/SelectFieldAppendCorrectorTest.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.chat.query.llm.dsl.corrector;
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
import com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector;
import org.junit.Assert;
import org.junit.jupiter.api.Test;

Some files were not shown because too many files have changed in this diff Show More