diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index e2914f706..1c7d8d006 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -467,40 +467,28 @@ public class QueryServiceImpl implements QueryService { return; } for (QueryFilter dslQueryFilter : metricFilters) { - Map map = new HashMap<>(); for (FilterExpression filterExpression : filterExpressionList) { if (filterExpression.getFieldName() != null && filterExpression.getFieldName().contains(dslQueryFilter.getName())) { - if (dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator()) - && Objects.nonNull(dslQueryFilter.getValue())) { - map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); - filedNameToValueMap.put(dslQueryFilter.getName(), map); - contextMetricFilters.stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); - } - }); - } else { - removeFieldNames.add(dslQueryFilter.getName()); - if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { - EqualsTo equalsTo = new EqualsTo(); - addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions); - } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) { - GreaterThanEquals greaterThanEquals = new GreaterThanEquals(); - addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions); - } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) { - GreaterThan greaterThan = new GreaterThan(); - addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions); - } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) { - MinorThanEquals minorThanEquals = new MinorThanEquals(); - addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions); - } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) { - MinorThan minorThan = new MinorThan(); - addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions); - } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) { - InExpression inExpression = new InExpression(); - addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions); - } + removeFieldNames.add(dslQueryFilter.getName()); + if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { + EqualsTo equalsTo = new EqualsTo(); + addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions); + } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) { + GreaterThanEquals greaterThanEquals = new GreaterThanEquals(); + addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions); + } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) { + GreaterThan greaterThan = new GreaterThan(); + addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions); + } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) { + MinorThanEquals minorThanEquals = new MinorThanEquals(); + addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions); + } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) { + MinorThan minorThan = new MinorThan(); + addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions); + } else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) { + InExpression inExpression = new InExpression(); + addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions); } break; } @@ -548,9 +536,14 @@ public class QueryServiceImpl implements QueryService { return; } Column column = new Column(columnName); - LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString())); comparisonExpression.setLeftExpression(column); - comparisonExpression.setRightExpression(longValue); + if (StringUtils.isNumeric(dslQueryFilter.getValue().toString())) { + LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString())); + comparisonExpression.setRightExpression(longValue); + } else { + StringValue stringValue = new StringValue(dslQueryFilter.getValue().toString()); + comparisonExpression.setRightExpression(stringValue); + } addConditions.add(comparisonExpression); contextMetricFilters.stream().forEach(o -> { if (o.getName().equals(dslQueryFilter.getName())) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java index 5f26cac4b..d9e4b2eec 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java @@ -1,7 +1,11 @@ package com.tencent.supersonic.common.util.jsqlparser; +import java.util.List; +import java.util.ArrayList; import java.util.Map; import java.util.Objects; + +import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.DoubleValue; import net.sf.jsqlparser.expression.Expression; @@ -15,6 +19,8 @@ import net.sf.jsqlparser.expression.operators.relational.GreaterThan; import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; import net.sf.jsqlparser.expression.operators.relational.MinorThan; import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.schema.Column; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -52,6 +58,28 @@ public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter { replaceComparisonExpression(expr); } + public void visit(InExpression inExpression) { + Column column = (Column) inExpression.getLeftExpression(); + Map valueMap = filedNameToValueMap.get(column.getColumnName()); + ExpressionList rightItemsList = (ExpressionList) inExpression.getRightItemsList(); + List expressions = rightItemsList.getExpressions(); + List values = new ArrayList<>(); + expressions.stream().forEach(o -> { + if (o instanceof StringValue) { + values.add(((StringValue) o).getValue()); + } + }); + String value = valueMap.get(JsonUtil.toString(values)); + List valueList = JsonUtil.toList(value, String.class); + List newExpressions = new ArrayList<>(); + valueList.stream().forEach(o -> { + StringValue stringValue = new StringValue(o); + newExpressions.add(stringValue); + }); + rightItemsList.setExpressions(newExpressions); + inExpression.setRightItemsList(rightItemsList); + } + public void replaceComparisonExpression(T expression) { Expression leftExpression = ((ComparisonOperator) expression).getLeftExpression(); Expression rightExpression = ((ComparisonOperator) expression).getRightExpression(); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/JsqlConstants.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/JsqlConstants.java index aad09606c..61b336697 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/JsqlConstants.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/JsqlConstants.java @@ -16,6 +16,7 @@ public class JsqlConstants { public static final String EQUAL_CONSTANT = " 1 = 1 "; public static final String IN_CONSTANT = " 1 in (1) "; + public static final String LIKE_CONSTANT = "'a' like 'a'"; public static final String IN = "IN"; } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java index 7eda5f260..8707c6d13 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java @@ -14,6 +14,7 @@ import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; import net.sf.jsqlparser.expression.operators.relational.InExpression; import net.sf.jsqlparser.expression.operators.relational.MinorThan; import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; +import net.sf.jsqlparser.expression.operators.relational.LikeExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; @@ -118,6 +119,22 @@ public class SqlParserRemoveHelper { log.error("JSQLParserException", e); } } + if (expression instanceof LikeExpression) { + LikeExpression likeExpression = (LikeExpression) expression; + String columnName = SqlParserSelectHelper.getColumnName(likeExpression.getLeftExpression(), + likeExpression.getRightExpression()); + if (!removeFieldNames.contains(columnName)) { + return; + } + try { + LikeExpression constantExpression = (LikeExpression) CCJSqlParserUtil.parseCondExpression( + JsqlConstants.LIKE_CONSTANT); + likeExpression.setLeftExpression(constantExpression.getLeftExpression()); + likeExpression.setRightExpression(constantExpression.getRightExpression()); + } catch (JSQLParserException e) { + log.error("JSQLParserException", e); + } + } } public static String removeHavingCondition(String sql, Set removeFieldNames) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java index 740db6c84..418df7b60 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java @@ -1,9 +1,14 @@ package com.tencent.supersonic.semantic.query.utils; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -12,7 +17,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Objects; +import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -46,8 +54,59 @@ public class DimValueAspect { } Object[] args = joinPoint.getArgs(); QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0]; - + String sql = queryS2QLReq.getSql(); + log.info("correctorSql before replacing:{}", sql); + List filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); List dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId()); + Set fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet()); + Map> filedNameToValueMap = new HashMap<>(); + filterExpressionList.stream().forEach(expression -> { + if (fieldNames.contains(expression.getFieldName())) { + dimensions.stream().forEach(dimension -> { + if (expression.getFieldName().equals(dimension.getName())) { + if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue()) + && !CollectionUtils.isEmpty(dimension.getDimValueMaps())) { + dimension.getDimValueMaps().stream().forEach(dimValue -> { + if (!CollectionUtils.isEmpty(dimValue.getAlias()) + && dimValue.getAlias().contains(expression.getFieldValue().toString())) { + Map map = new HashMap<>(); + map.put(expression.getFieldValue().toString(), dimValue.getTechName()); + filedNameToValueMap.put(expression.getFieldName(), map); + } + }); + } + if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) { + String fieldValue = JsonUtil.toString(expression.getFieldValue()); + fieldValue = fieldValue.replace("'", ""); + List values = JsonUtil.toList(fieldValue, String.class); + List revisedValues = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + Boolean flag = new Boolean(false); + for (DimValueMap dimValueMap : dimension.getDimValueMaps()) { + if (dimValueMap.getAlias().contains(values.get(i))) { + flag = true; + revisedValues.add(dimValueMap.getTechName()); + break; + } + } + if (!flag) { + revisedValues.add(values.get(i)); + } + } + if (!revisedValues.equals(values)) { + Map map = new HashMap<>(); + map.put(JsonUtil.toString(values), JsonUtil.toString(revisedValues)); + filedNameToValueMap.put(expression.getFieldName(), map); + } + } + } + }); + } + }); + log.info("filedNameToValueMap:{}", filedNameToValueMap); + sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); + log.info("correctorSql after replacing:{}", sql); + queryS2QLReq.setSql(sql); Map> techNameToBizName = getTechNameToBizName(dimensions); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();