topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) fix in replace bug (#302)

Browse Source
This commit is contained in:
mainmain
2023-10-31 15:54:15 +08:00
committed by GitHub
parent 688d26c457
commit ae9aa1ba0f
5 changed files with 111 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java
View File

@@ -93,4 +93,4 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
String sql = SqlParserReplaceHelper.replaceValue(semanticCorrectInfo.getSql(), filedNameToValueMap, false);
semanticCorrectInfo.setSql(sql);
}
}
}

69
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.service.impl;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -34,6 +35,7 @@ import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.chat.service.StatisticsService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.util.ContextUtils;
@@ -62,6 +64,8 @@ import java.util.Objects;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
@@ -247,7 +251,7 @@ public class QueryServiceImpl implements QueryService {
queryResult.setQueryTimeCost(System.currentTimeMillis() - executeTime);
return queryResult;
}
// save time cost data
public void saveInfo(List<StatisticsDO> timeCostDOList,
String queryText, Long queryId,
String userName, Long chatId) {
@@ -324,7 +328,8 @@ public class QueryServiceImpl implements QueryService {
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
return context.getParseInfo();
}
//mainly used for executing after revising filters,for example:"fans_cnt>=100000"->"fans_cnt>500000",
//"style='流行'"->"style in ['流行','爱国']"
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException {
ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(),
@@ -339,19 +344,21 @@ public class QueryServiceImpl implements QueryService {
String correctorSql = parseInfo.getSqlInfo().getLogicSql();
log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter
List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
List<FilterExpression> havingExpressionList = SqlParserSelectHelper.getHavingExpressions(correctorSql);
List<Expression> addWhereConditions = new ArrayList<>();
List<Expression> addHavingConditions = new ArrayList<>();
Set<String> removeWhereFieldNames = new HashSet<>();
Set<String> removeHavingFieldNames = new HashSet<>();
// replace where filter
updateFilters(filedNameToValueMap, whereExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames);
updateDateInfo(queryData, parseInfo, filedNameToValueMap,
whereExpressionList, addWhereConditions, removeWhereFieldNames);
correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames);
// replace having filter
updateFilters(havingFiledNameToValueMap, havingExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames);
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
@@ -399,15 +406,18 @@ public class QueryServiceImpl implements QueryService {
queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1));
}
// startDate equals to endDate
if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
for (FilterExpression filterExpression : filterExpressionList) {
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
//sql where condition exists 'equals' operator about date,just replace
if (filterExpression.getOperator().equals(FilterOperatorEnum.EQUALS)) {
dateField = filterExpression.getFieldName();
map.put(filterExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
filedNameToValueMap.put(dateField, map);
} else {
// first remove,then add
removeFieldNames.add(DateUtils.DATE_FIELD);
EqualsTo equalsTo = new EqualsTo();
Column column = new Column(DateUtils.DATE_FIELD);
@@ -423,6 +433,7 @@ public class QueryServiceImpl implements QueryService {
for (FilterExpression filterExpression : filterExpressionList) {
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
dateField = filterExpression.getFieldName();
//just replace
if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue().equals(filterExpression.getOperator())
|| FilterOperatorEnum.GREATER_THAN.getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(),
@@ -434,12 +445,13 @@ public class QueryServiceImpl implements QueryService {
queryData.getDateInfo().getEndDate());
}
filedNameToValueMap.put(dateField, map);
// first remove,then add
if (FilterOperatorEnum.EQUALS.getValue().equals(filterExpression.getOperator())) {
removeFieldNames.add(DateUtils.DATE_FIELD);
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeCondition(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeCondition(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
}
}
}
@@ -447,7 +459,7 @@ public class QueryServiceImpl implements QueryService {
parseInfo.setDateInfo(queryData.getDateInfo());
}
public <T extends ComparisonOperator> void addTimeCondition(String date,
public <T extends ComparisonOperator> void addTimeFilters(String date,
T comparisonExpression,
List<Expression> addConditions) {
Column column = new Column(DateUtils.DATE_FIELD);
@@ -473,30 +485,30 @@ public class QueryServiceImpl implements QueryService {
removeFieldNames.add(dslQueryFilter.getName());
if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
EqualsTo equalsTo = new EqualsTo();
addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) {
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) {
GreaterThan greaterThan = new GreaterThan();
addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) {
MinorThanEquals minorThanEquals = new MinorThanEquals();
addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) {
MinorThan minorThan = new MinorThan();
addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
addWhereFilters(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) {
InExpression inExpression = new InExpression();
addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
addWhereInFilters(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
}
break;
}
}
}
}
public void addWhereInCondition(QueryFilter dslQueryFilter,
// add in condition to sql where condition
public void addWhereInFilters(QueryFilter dslQueryFilter,
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
@@ -523,8 +535,8 @@ public class QueryServiceImpl implements QueryService {
}
});
}
public <T extends ComparisonOperator> void addWhereCondition(QueryFilter dslQueryFilter,
// add where filter
public <T extends ComparisonOperator> void addWhereFilters(QueryFilter dslQueryFilter,
T comparisonExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
@@ -580,7 +592,11 @@ public class QueryServiceImpl implements QueryService {
Set<Long> detectModelIds = new HashSet<>();
detectModelIds.add(dimensionValueReq.getModelId());
List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds);
// if the search results is null,search dimensionValue from database
if (CollectionUtils.isEmpty(dimensionValues)) {
queryResultWithSchemaResp = queryDatabase(dimensionValueReq, user);
return queryResultWithSchemaResp;
}
List<QueryColumn> columns = new ArrayList<>();
QueryColumn queryColumn = new QueryColumn();
queryColumn.setNameEn(dimensionValueReq.getBizName());
@@ -628,5 +644,24 @@ public class QueryServiceImpl implements QueryService {
.collect(Collectors.toList());
}
private QueryResultWithSchemaResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateConf.DateMode.RECENT);
dateConf.setUnit(1);
dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
queryStructReq.setModelId(dimensionValueReq.getModelId());
queryStructReq.setNativeQuery(false);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
return queryResultWithSchemaResp;
}
}

9
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java
View File

@@ -59,6 +59,9 @@ public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter {
}
public void visit(InExpression inExpression) {
if (!(inExpression.getLeftExpression() instanceof Column)) {
return;
}
Column column = (Column) inExpression.getLeftExpression();
Map<String, String> valueMap = filedNameToValueMap.get(column.getColumnName());
ExpressionList rightItemsList = (ExpressionList) inExpression.getRightItemsList();
@@ -69,7 +72,13 @@ public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter {
values.add(((StringValue) o).getValue());
}
});
if (valueMap == null) {
return;
}
String value = valueMap.get(JsonUtil.toString(values));
if (StringUtils.isBlank(value)) {
return;
}
List<String> valueList = JsonUtil.toList(value, String.class);
List<Expression> newExpressions = new ArrayList<>();
valueList.stream().forEach(o -> {

5
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.statement.select.Select;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
@@ -13,6 +14,10 @@ class SqlParserSelectHelperTest {
@Test
void getWhereFilterExpression() {
Select selectStatement = SqlParserSelectHelper.getSelect(
"select 用户名, 访问次数 from 超音数 where 用户名 in ('alice', 'lucy')");
System.out.println(selectStatement);
List<FilterExpression> filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE "
+ "sys_imp_date = '2023-08-08' AND YEAR(publish_date) = 2023 "

74
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -56,6 +56,7 @@ public class DimValueAspect {
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
String sql = queryS2QLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
@@ -63,42 +64,20 @@ public class DimValueAspect {
filterExpressionList.stream().forEach(expression -> {
if (fieldNames.contains(expression.getFieldName())) {
dimensions.stream().forEach(dimension -> {
if (expression.getFieldName().equals(dimension.getName())) {
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
if (expression.getFieldName().equals(dimension.getName())
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
// consider '=' filter
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
Map<String, String> map = new HashMap<>();
map.put(expression.getFieldValue().toString(), dimValue.getTechName());
filedNameToValueMap.put(expression.getFieldName(), map);
getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(),
dimValue.getTechName(), expression.getFieldName());
}
});
}
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
fieldValue = fieldValue.replace("'", "");
List<String> values = JsonUtil.toList(fieldValue, String.class);
List<String> revisedValues = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
Boolean flag = new Boolean(false);
for (DimValueMap dimValueMap : dimension.getDimValueMaps()) {
if (dimValueMap.getAlias().contains(values.get(i))) {
flag = true;
revisedValues.add(dimValueMap.getTechName());
break;
}
}
if (!flag) {
revisedValues.add(values.get(i));
}
}
if (!revisedValues.equals(values)) {
Map<String, String> map = new HashMap<>();
map.put(JsonUtil.toString(values), JsonUtil.toString(revisedValues));
filedNameToValueMap.put(expression.getFieldName(), map);
}
}
// consider 'in' filter,each element needs to judge.
replaceInCondition(expression, dimension, filedNameToValueMap);
}
});
}
@@ -116,6 +95,41 @@ public class DimValueAspect {
return queryResultWithColumns;
}
public void replaceInCondition(FilterExpression expression, DimensionResp dimension,
Map<String, Map<String, String>> filedNameToValueMap) {
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
fieldValue = fieldValue.replace("'", "");
List<String> values = JsonUtil.toList(fieldValue, String.class);
List<String> revisedValues = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
Boolean flag = new Boolean(false);
for (DimValueMap dimValueMap : dimension.getDimValueMaps()) {
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())
&& dimValueMap.getAlias().contains(values.get(i))) {
flag = true;
revisedValues.add(dimValueMap.getTechName());
break;
}
}
if (!flag) {
revisedValues.add(values.get(i));
}
}
if (!revisedValues.equals(values)) {
getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values),
JsonUtil.toString(revisedValues), expression.getFieldName());
}
}
}
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
String oldValue, String newValue, String fieldName) {
Map<String, String> map = new HashMap<>();
map.put(oldValue, newValue);
filedNameToValueMap.put(fieldName, map);
}
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))"