topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement] add dimensionValue alias for sql execute (#296)

Browse Source
This commit is contained in:
mainmain
2023-10-26 21:13:43 +08:00
committed by GitHub
parent c92184d89f
commit 80cce47f58
5 changed files with 132 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -467,40 +467,28 @@ public class QueryServiceImpl implements QueryService {
return;
}
for (QueryFilter dslQueryFilter : metricFilters) {
Map<String, String> map = new HashMap<>();
for (FilterExpression filterExpression : filterExpressionList) {
if (filterExpression.getFieldName() != null
&& filterExpression.getFieldName().contains(dslQueryFilter.getName())) {
if (dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())
&& Objects.nonNull(dslQueryFilter.getValue())) {
map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString());
filedNameToValueMap.put(dslQueryFilter.getName(), map);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
}
});
} else {
removeFieldNames.add(dslQueryFilter.getName());
if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
EqualsTo equalsTo = new EqualsTo();
addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) {
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) {
GreaterThan greaterThan = new GreaterThan();
addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) {
MinorThanEquals minorThanEquals = new MinorThanEquals();
addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) {
MinorThan minorThan = new MinorThan();
addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) {
InExpression inExpression = new InExpression();
addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
}
removeFieldNames.add(dslQueryFilter.getName());
if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
EqualsTo equalsTo = new EqualsTo();
addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) {
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) {
GreaterThan greaterThan = new GreaterThan();
addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) {
MinorThanEquals minorThanEquals = new MinorThanEquals();
addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) {
MinorThan minorThan = new MinorThan();
addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) {
InExpression inExpression = new InExpression();
addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
}
break;
}
@@ -548,9 +536,14 @@ public class QueryServiceImpl implements QueryService {
return;
}
Column column = new Column(columnName);
LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString()));
comparisonExpression.setLeftExpression(column);
comparisonExpression.setRightExpression(longValue);
if (StringUtils.isNumeric(dslQueryFilter.getValue().toString())) {
LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString()));
comparisonExpression.setRightExpression(longValue);
} else {
StringValue stringValue = new StringValue(dslQueryFilter.getValue().toString());
comparisonExpression.setRightExpression(stringValue);
}
addConditions.add(comparisonExpression);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {

28
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java
View File

@@ -1,7 +1,11 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.Objects;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
@@ -15,6 +19,8 @@ import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -52,6 +58,28 @@ public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter {
replaceComparisonExpression(expr);
}
public void visit(InExpression inExpression) {
Column column = (Column) inExpression.getLeftExpression();
Map<String, String> valueMap = filedNameToValueMap.get(column.getColumnName());
ExpressionList rightItemsList = (ExpressionList) inExpression.getRightItemsList();
List<Expression> expressions = rightItemsList.getExpressions();
List<String> values = new ArrayList<>();
expressions.stream().forEach(o -> {
if (o instanceof StringValue) {
values.add(((StringValue) o).getValue());
}
});
String value = valueMap.get(JsonUtil.toString(values));
List<String> valueList = JsonUtil.toList(value, String.class);
List<Expression> newExpressions = new ArrayList<>();
valueList.stream().forEach(o -> {
StringValue stringValue = new StringValue(o);
newExpressions.add(stringValue);
});
rightItemsList.setExpressions(newExpressions);
inExpression.setRightItemsList(rightItemsList);
}
public <T extends Expression> void replaceComparisonExpression(T expression) {
Expression leftExpression = ((ComparisonOperator) expression).getLeftExpression();
Expression rightExpression = ((ComparisonOperator) expression).getRightExpression();

1
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/JsqlConstants.java
View File

@@ -16,6 +16,7 @@ public class JsqlConstants {
public static final String EQUAL_CONSTANT = " 1 = 1 ";
public static final String IN_CONSTANT = " 1 in (1) ";
public static final String LIKE_CONSTANT = "'a' like 'a'";
public static final String IN = "IN";
}

17
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java
View File

@@ -14,6 +14,7 @@ import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.expression.operators.relational.LikeExpression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
@@ -118,6 +119,22 @@ public class SqlParserRemoveHelper {
log.error("JSQLParserException", e);
}
}
if (expression instanceof LikeExpression) {
LikeExpression likeExpression = (LikeExpression) expression;
String columnName = SqlParserSelectHelper.getColumnName(likeExpression.getLeftExpression(),
likeExpression.getRightExpression());
if (!removeFieldNames.contains(columnName)) {
return;
}
try {
LikeExpression constantExpression = (LikeExpression) CCJSqlParserUtil.parseCondExpression(
JsqlConstants.LIKE_CONSTANT);
likeExpression.setLeftExpression(constantExpression.getLeftExpression());
likeExpression.setRightExpression(constantExpression.getRightExpression());
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
}
}
public static String removeHavingCondition(String sql, Set<String> removeFieldNames) {

61
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -1,9 +1,14 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -12,7 +17,10 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -46,8 +54,59 @@ public class DimValueAspect {
}
Object[] args = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
String sql = queryS2QLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
filterExpressionList.stream().forEach(expression -> {
if (fieldNames.contains(expression.getFieldName())) {
dimensions.stream().forEach(dimension -> {
if (expression.getFieldName().equals(dimension.getName())) {
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
Map<String, String> map = new HashMap<>();
map.put(expression.getFieldValue().toString(), dimValue.getTechName());
filedNameToValueMap.put(expression.getFieldName(), map);
}
});
}
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
fieldValue = fieldValue.replace("'", "");
List<String> values = JsonUtil.toList(fieldValue, String.class);
List<String> revisedValues = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
Boolean flag = new Boolean(false);
for (DimValueMap dimValueMap : dimension.getDimValueMaps()) {
if (dimValueMap.getAlias().contains(values.get(i))) {
flag = true;
revisedValues.add(dimValueMap.getTechName());
break;
}
}
if (!flag) {
revisedValues.add(values.get(i));
}
}
if (!revisedValues.equals(values)) {
Map<String, String> map = new HashMap<>();
map.put(JsonUtil.toString(values), JsonUtil.toString(revisedValues));
filedNameToValueMap.put(expression.getFieldName(), map);
}
}
}
});
}
});
log.info("filedNameToValueMap:{}", filedNameToValueMap);
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
queryS2QLReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();