topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Reduce the two calls to the mapper in the parser stage. (#888)

Browse Source
This commit is contained in:
lexluo09
2024-04-06 11:49:44 +08:00
committed by GitHub
parent 0577090b39
commit faeb5bbeac
4 changed files with 34 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java
View File

@@ -4,6 +4,9 @@ import com.tencent.supersonic.chat.server.agent.Agent;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import org.apache.commons.collections.MapUtils;
import java.util.Objects;
public class QueryReqConverter {
@@ -18,6 +21,10 @@ public class QueryReqConverter {
queryReq.setEnableLLM(true);
}
queryReq.setDataSetIds(agent.getDataSetIds());
if (Objects.nonNull(queryReq.getMapInfo())
&& MapUtils.isNotEmpty(queryReq.getMapInfo().getDataSetElementMatches())) {
queryReq.setMapInfo(queryReq.getMapInfo());
}
return queryReq;
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import lombok.Data;
import java.util.Set;
@@ -15,4 +16,5 @@ public class QueryReq {
private QueryFilters queryFilters;
private boolean saveAnswer = true;
private boolean enableLLM;
private SchemaMapInfo mapInfo = new SchemaMapInfo();
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java
View File

@@ -40,7 +40,7 @@ public class OptimizationConfig {
@Value("${embedding.mapper.word.min:4}")
private int embeddingMapperWordMin;
@Value("${embedding.mapper.word.max:5}")
@Value("${embedding.mapper.word.max:4}")
private int embeddingMapperWordMax;
@Value("${embedding.mapper.batch:50}")

44
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -72,6 +72,7 @@ import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
@@ -129,9 +130,12 @@ public class ChatQueryServiceImpl implements ChatQueryService {
ChatContext chatCtx = chatContextService.getOrCreateContext(queryReq.getChatId());
// 1. mapper
schemaMappers.forEach(mapper -> {
mapper.map(queryCtx);
});
if (Objects.isNull(queryReq.getMapInfo())
|| MapUtils.isEmpty(queryReq.getMapInfo().getDataSetElementMatches())) {
schemaMappers.forEach(mapper -> {
mapper.map(queryCtx);
});
}
// 2. parser
semanticParsers.forEach(parser -> {
@@ -209,7 +213,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
private QueryResult doExecution(SemanticQueryReq semanticQueryReq,
SemanticParseInfo parseInfo, User user) throws Exception {
SemanticParseInfo parseInfo, User user) throws Exception {
SemanticQueryResp queryResp = queryService.queryByReq(semanticQueryReq, user);
QueryResult queryResult = new QueryResult();
if (queryResp != null) {
@@ -357,10 +361,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo,
Map<String, Map<String, String>> filedNameToValueMap,
List<FieldExpression> fieldExpressionList,
List<Expression> addConditions,
Set<String> removeFieldNames) {
Map<String, Map<String, String>> filedNameToValueMap,
List<FieldExpression> fieldExpressionList,
List<Expression> addConditions,
Set<String> removeFieldNames) {
if (Objects.isNull(queryData.getDateInfo())) {
return;
}
@@ -424,8 +428,8 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
private <T extends ComparisonOperator> void addTimeFilters(String date,
T comparisonExpression,
List<Expression> addConditions) {
T comparisonExpression,
List<Expression> addConditions) {
Column column = new Column(TimeDimensionEnum.DAY.getChName());
StringValue stringValue = new StringValue(date);
comparisonExpression.setLeftExpression(column);
@@ -434,10 +438,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
private void updateFilters(List<FieldExpression> fieldExpressionList,
Set<QueryFilter> metricFilters,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions,
Set<String> removeFieldNames) {
Set<QueryFilter> metricFilters,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions,
Set<String> removeFieldNames) {
if (CollectionUtils.isEmpty(metricFilters)) {
return;
}
@@ -473,9 +477,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
// add in condition to sql where condition
private void addWhereInFilters(QueryFilter dslQueryFilter,
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
Column column = new Column(dslQueryFilter.getName());
ExpressionList expressionList = new ExpressionList();
List<Expression> expressions = new ArrayList<>();
@@ -502,9 +506,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
// add where filter
private <T extends ComparisonOperator> void addWhereFilters(QueryFilter dslQueryFilter,
T comparisonExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
T comparisonExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
String columnName = dslQueryFilter.getName();
if (StringUtils.isNotBlank(dslQueryFilter.getFunction())) {
columnName = dslQueryFilter.getFunction() + "(" + dslQueryFilter.getName() + ")";