topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) fix in replace bug (#302)

Browse Source
This commit is contained in:
mainmain
2023-10-31 15:54:15 +08:00
committed by GitHub
parent 688d26c457
commit ae9aa1ba0f
5 changed files with 111 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java
View File

@@ -93,4 +93,4 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
String sql = SqlParserReplaceHelper.replaceValue(semanticCorrectInfo.getSql(), filedNameToValueMap, false);
semanticCorrectInfo.setSql(sql);
}
}
}

69
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.service.impl;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -34,6 +35,7 @@ import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.service.StatisticsService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.util.ContextUtils;
@@ -62,6 +64,8 @@ import java.util.Objects;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
@@ -247,7 +251,7 @@ public class QueryServiceImpl implements QueryService {
queryResult.setQueryTimeCost(System.currentTimeMillis() - executeTime);
return queryResult;
}
// save time cost data
public void saveInfo(List<StatisticsDO> timeCostDOList,
String queryText, Long queryId,
String userName, Long chatId) {
@@ -324,7 +328,8 @@ public class QueryServiceImpl implements QueryService {
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
return context.getParseInfo();
}
//mainly used for executing after revising filters,for example:"fans_cnt>=100000"->"fans_cnt>500000",
//"style='流行'"->"style in ['流行','爱国']"
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException {
ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(),
@@ -339,19 +344,21 @@ public class QueryServiceImpl implements QueryService {
String correctorSql = parseInfo.getSqlInfo().getLogicSql();
log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter
List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
List<FilterExpression> havingExpressionList = SqlParserSelectHelper.getHavingExpressions(correctorSql);
List<Expression> addWhereConditions = new ArrayList<>();
List<Expression> addHavingConditions = new ArrayList<>();
Set<String> removeWhereFieldNames = new HashSet<>();
Set<String> removeHavingFieldNames = new HashSet<>();
// replace where filter
updateFilters(filedNameToValueMap, whereExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames);
updateDateInfo(queryData, parseInfo, filedNameToValueMap,
whereExpressionList, addWhereConditions, removeWhereFieldNames);
correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames);
// replace having filter
updateFilters(havingFiledNameToValueMap, havingExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames);
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
@@ -399,15 +406,18 @@ public class QueryServiceImpl implements QueryService {
queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1));
}
// startDate equals to endDate
if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
for (FilterExpression filterExpression : filterExpressionList) {
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
//sql where condition exists 'equals' operator about date,just replace
if (filterExpression.getOperator().equals(FilterOperatorEnum.EQUALS)) {
dateField = filterExpression.getFieldName();
map.put(filterExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
filedNameToValueMap.put(dateField, map);
} else {
// first remove,then add
removeFieldNames.add(DateUtils.DATE_FIELD);
EqualsTo equalsTo = new EqualsTo();
Column column = new Column(DateUtils.DATE_FIELD);
@@ -423,6 +433,7 @@ public class QueryServiceImpl implements QueryService {
for (FilterExpression filterExpression : filterExpressionList) {
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
dateField = filterExpression.getFieldName();
//just replace
if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue().equals(filterExpression.getOperator())
|| FilterOperatorEnum.GREATER_THAN.getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(),
@@ -434,12 +445,13 @@ public class QueryServiceImpl implements QueryService {
queryData.getDateInfo().getEndDate());
}
filedNameToValueMap.put(dateField, map);
// first remove,then add
if (FilterOperatorEnum.EQUALS.getValue().equals(filterExpression.getOperator())) {
removeFieldNames.add(DateUtils.DATE_FIELD);
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeCondition(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeCondition(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
}
}
}
@@ -447,7 +459,7 @@ public class QueryServiceImpl implements QueryService {
parseInfo.setDateInfo(queryData.getDateInfo());
}
public <T extends ComparisonOperator> void addTimeCondition(String date,
public <T extends ComparisonOperator> void addTimeFilters(String date,
T comparisonExpression,
List<Expression> addConditions) {
Column column = new Column(DateUtils.DATE_FIELD);
@@ -473,30 +485,30 @@ public class QueryServiceImpl implements QueryService {
removeFieldNames.add(dslQueryFilter.getName());
if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
EqualsTo equalsTo = new EqualsTo();
addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) {
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) {
GreaterThan greaterThan = new GreaterThan();
addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) {
MinorThanEquals minorThanEquals = new MinorThanEquals();
addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) {
MinorThan minorThan = new MinorThan();
addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) {
InExpression inExpression = new InExpression();
addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
addWhereInFilters(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
}
break;
}
}
}
}
public void addWhereInCondition(QueryFilter dslQueryFilter,
// add in condition to sql where condition
public void addWhereInFilters(QueryFilter dslQueryFilter,
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
@@ -523,8 +535,8 @@ public class QueryServiceImpl implements QueryService {
}
});
}
public <T extends ComparisonOperator> void addWhereCondition(QueryFilter dslQueryFilter,
// add where filter
public <T extends ComparisonOperator> void addWhereFilters(QueryFilter dslQueryFilter,
T comparisonExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
@@ -580,7 +592,11 @@ public class QueryServiceImpl implements QueryService {
Set<Long> detectModelIds = new HashSet<>();
detectModelIds.add(dimensionValueReq.getModelId());
List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds);
// if the search results is null,search dimensionValue from database
if (CollectionUtils.isEmpty(dimensionValues)) {
queryResultWithSchemaResp = queryDatabase(dimensionValueReq, user);
return queryResultWithSchemaResp;
}
List<QueryColumn> columns = new ArrayList<>();
QueryColumn queryColumn = new QueryColumn();
queryColumn.setNameEn(dimensionValueReq.getBizName());
@@ -628,5 +644,24 @@ public class QueryServiceImpl implements QueryService {
.collect(Collectors.toList());
}
private QueryResultWithSchemaResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateConf.DateMode.RECENT);
dateConf.setUnit(1);
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
queryStructReq.setModelId(dimensionValueReq.getModelId());
queryStructReq.setNativeQuery(false);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
return queryResultWithSchemaResp;
}
}