topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Do not modify the values in the like expression and optimize some of the jsqlparser code. (#1584)

Browse Source
This commit is contained in:
lexluo09
2024-08-19 16:58:19 +08:00
committed by GitHub
parent ba55ecb31e
commit bd2eaef3f6
9 changed files with 100 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java
View File

@@ -1,9 +1,5 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -20,6 +16,11 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@Slf4j
public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
@@ -76,37 +77,39 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
public List<Expression> parserFilter(ComparisonOperator comparisonOperator, String condExpr) {
List<Expression> result = new ArrayList<>();
String toString = comparisonOperator.toString();
String comparisonOperatorStr = comparisonOperator.toString();
Expression leftExpression = comparisonOperator.getLeftExpression();
if (!(leftExpression instanceof Function)) {
return result;
}
Function leftExpressionFunction = (Function) leftExpression;
if (leftExpressionFunction.toString().contains(JsqlConstants.DATE_FUNCTION)) {
Function leftFunction = (Function) leftExpression;
if (leftFunction.toString().contains(JsqlConstants.DATE_FUNCTION)) {
return result;
}
//List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
ExpressionList<?> leftExpressions = leftExpressionFunction.getParameters();
if (CollectionUtils.isEmpty(leftExpressions)) {
ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
if (CollectionUtils.isEmpty(leftFunctionParams)) {
return result;
}
Column field = (Column) leftExpressions.get(0);
Column field = (Column) leftFunctionParams.get(0);
String columnName = field.getColumnName();
if (!fieldNames.contains(columnName)) {
return null;
}
try {
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
comparisonOperator.setLeftExpression(expression.getLeftExpression());
comparisonOperator.setRightExpression(expression.getRightExpression());
comparisonOperator.setASTNode(expression.getASTNode());
result.add(CCJSqlParserUtil.parseCondExpression(toString));
ComparisonOperator parsedExpression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
comparisonOperator.setLeftExpression(parsedExpression.getLeftExpression());
comparisonOperator.setRightExpression(parsedExpression.getRightExpression());
comparisonOperator.setASTNode(parsedExpression.getASTNode());
result.add(CCJSqlParserUtil.parseCondExpression(comparisonOperatorStr));
return result;
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
return null;
}
}

45
common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledNameReplaceVisitor.java
View File

@@ -1,8 +1,5 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
@@ -12,9 +9,11 @@ import net.sf.jsqlparser.expression.operators.relational.LikeExpression;
import net.sf.jsqlparser.schema.Column;
import org.springframework.util.CollectionUtils;
public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter {
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public static final String PREFIX = "%";
public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, Set<String>> fieldValueToFieldNames;
public FiledNameReplaceVisitor(Map<String, Set<String>> fieldValueToFieldNames) {
@@ -34,40 +33,20 @@ public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter {
private void replaceFieldNameByFieldValue(BinaryExpression expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
if (!(rightExpression instanceof StringValue)) {
if (!(rightExpression instanceof StringValue) || !(leftExpression instanceof Column)
|| CollectionUtils.isEmpty(fieldValueToFieldNames)
|| Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
return;
}
if (!(leftExpression instanceof Column)) {
return;
}
if (CollectionUtils.isEmpty(fieldValueToFieldNames)) {
return;
}
if (Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
return;
}
Column leftColumnName = (Column) leftExpression;
Column leftColumn = (Column) leftExpression;
StringValue rightStringValue = (StringValue) rightExpression;
if (expr instanceof LikeExpression) {
String value = getValue(rightStringValue.getValue());
rightStringValue.setValue(value);
}
Set<String> fieldNames = fieldValueToFieldNames.get(rightStringValue.getValue());
if (!CollectionUtils.isEmpty(fieldNames) && !fieldNames.contains(leftColumnName.getColumnName())) {
leftColumnName.setColumnName(fieldNames.stream().findFirst().get());
if (!CollectionUtils.isEmpty(fieldNames) && !fieldNames.contains(leftColumn.getColumnName())) {
leftColumn.setColumnName(fieldNames.stream().findFirst().get());
}
}
private String getValue(String value) {
if (value.startsWith(PREFIX)) {
value = value.substring(1);
}
if (value.endsWith(PREFIX)) {
value = value.substring(0, value.length() - 1);
}
return value;
}
}

94
common/src/main/java/com/tencent/supersonic/common/jsqlparser/FilterRemoveVisitor.java
View File

@@ -1,94 +0,0 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.List;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.lang3.StringUtils;
public class FilterRemoveVisitor extends ExpressionVisitorAdapter {
private List<String> filedNames;
public FilterRemoveVisitor(List<String> filedNames) {
this.filedNames = filedNames;
}
private boolean isRemove(Expression leftExpression) {
if (!(leftExpression instanceof Column)) {
return false;
}
Column leftColumnName = (Column) leftExpression;
String columnName = leftColumnName.getColumnName();
if (StringUtils.isEmpty(columnName)) {
return false;
}
if (!filedNames.contains(columnName)) {
return false;
}
return true;
}
@Override
public void visit(EqualsTo expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(MinorThan expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(0L));
}
@Override
public void visit(MinorThanEquals expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(GreaterThan expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(0L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(GreaterThanEquals expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(InExpression expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setNot(false);
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
}

11
common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionNameReplaceVisitor.java
View File

@@ -1,24 +1,21 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
@Slf4j
public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, String> functionMap;
private Map<String, UnaryOperator> functionCallMap;
public FunctionNameReplaceVisitor(Map<String, String> functionMap) {
this.functionMap = functionMap;
}
public FunctionNameReplaceVisitor(Map<String, String> functionMap, Map<String, UnaryOperator> functionCallMap) {
this.functionMap = functionMap;
this.functionCallMap = functionCallMap;

123
common/src/main/java/com/tencent/supersonic/common/jsqlparser/FunctionReplaceVisitor.java
View File

@@ -1,123 +0,0 @@
package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.util.StringUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
private List<Expression> waitingForAdds = new ArrayList<>();
@Override
public void visit(MinorThan expr) {
List<Expression> expressions = reparseDate(expr, ">");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(EqualsTo expr) {
List<Expression> expressions = reparseDate(expr, ">=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(MinorThanEquals expr) {
List<Expression> expressions = reparseDate(expr, ">=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(GreaterThan expr) {
List<Expression> expressions = reparseDate(expr, "<");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(GreaterThanEquals expr) {
List<Expression> expressions = reparseDate(expr, "<=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
public List<Expression> getWaitingForAdds() {
return waitingForAdds;
}
public List<Expression> reparseDate(ComparisonOperator comparisonOperator, String startDateOperator) {
List<Expression> result = new ArrayList<>();
Expression leftExpression = comparisonOperator.getLeftExpression();
if (!(leftExpression instanceof Function)) {
return result;
}
Function leftExpressionFunction = (Function) leftExpression;
if (!leftExpressionFunction.toString().contains(JsqlConstants.DATE_FUNCTION)) {
return result;
}
//List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
ExpressionList<?> leftExpressions = leftExpressionFunction.getParameters();
if (CollectionUtils.isEmpty(leftExpressions) || leftExpressions.size() < 3) {
return result;
}
Column field = (Column) leftExpressions.get(1);
String columnName = field.getColumnName();
try {
String startDateValue = DateFunctionHelper.getStartDateStr(comparisonOperator, leftExpressions);
String endDateValue = DateFunctionHelper.getEndDateValue(leftExpressions);
String endDateOperator = comparisonOperator.getStringExpression();
String condExpr =
columnName + StringUtil.getSpaceWrap(DateFunctionHelper.getEndDateOperator(comparisonOperator))
+ StringUtil.getCommaWrap(endDateValue);
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
String startDataCondExpr =
columnName + StringUtil.getSpaceWrap(startDateOperator) + StringUtil.getCommaWrap(startDateValue);
if (JsqlConstants.EQUAL.equalsIgnoreCase(endDateOperator)) {
result.add(CCJSqlParserUtil.parseCondExpression(condExpr));
expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(JsqlConstants.EQUAL_CONSTANT);
}
if (startDateOperator.equals("<=") || startDateOperator.equals("<")) {
comparisonOperator.setLeftExpression(new Column("1"));
comparisonOperator.setRightExpression(new LongValue(1));
comparisonOperator.setASTNode(null);
} else {
comparisonOperator.setLeftExpression(expression.getLeftExpression());
comparisonOperator.setRightExpression(expression.getRightExpression());
comparisonOperator.setASTNode(expression.getASTNode());
}
result.add(CCJSqlParserUtil.parseCondExpression(startDataCondExpr));
return result;
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
return null;
}
}

45
common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByFunctionReplaceVisitor.java
View File

@@ -1,9 +1,5 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
@@ -12,16 +8,17 @@ import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.GroupByVisitor;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
@Slf4j
public class GroupByFunctionReplaceVisitor implements GroupByVisitor {
private Map<String, String> functionMap;
private Map<String, UnaryOperator> functionCallMap;
public GroupByFunctionReplaceVisitor(Map<String, String> functionMap) {
this.functionMap = functionMap;
}
public GroupByFunctionReplaceVisitor(Map<String, String> functionMap, Map<String, UnaryOperator> functionCallMap) {
this.functionMap = functionMap;
this.functionCallMap = functionCallMap;
@@ -31,22 +28,22 @@ public class GroupByFunctionReplaceVisitor implements GroupByVisitor {
groupByElement.getGroupByExpressionList();
ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList();
List<Expression> groupByExpressions = groupByExpressionList.getExpressions();
for (int i = 0; i < groupByExpressions.size(); i++) {
Expression expression = groupByExpressions.get(i);
if (expression instanceof Function) {
Function function = (Function) expression;
String functionName = function.getName().toLowerCase();
String replaceName = functionMap.get(functionName);
if (StringUtils.isNotBlank(replaceName)) {
function.setName(replaceName);
if (Objects.nonNull(functionCallMap) && functionCallMap.containsKey(functionName)) {
Object ret = functionCallMap.get(functionName).apply(function.getParameters());
if (Objects.nonNull(ret) && ret instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) ret;
function.setParameters(expressionList);
}
}
for (Expression expression : groupByExpressions) {
if (!(expression instanceof Function)) {
continue;
}
Function function = (Function) expression;
String functionName = function.getName().toLowerCase();
String replaceName = functionMap.get(functionName);
if (StringUtils.isBlank(replaceName)) {
continue;
}
function.setName(replaceName);
if (Objects.nonNull(functionCallMap) && functionCallMap.containsKey(functionName)) {
Object ret = functionCallMap.get(functionName).apply(function.getParameters());
if (Objects.nonNull(ret) && ret instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) ret;
function.setParameters(expressionList);
}
}
}

70
common/src/main/java/com/tencent/supersonic/common/jsqlparser/GroupByReplaceVisitor.java
View File

@@ -1,8 +1,5 @@
package com.tencent.supersonic.common.jsqlparser;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -14,6 +11,10 @@ import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.GroupByVisitor;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Slf4j
public class GroupByReplaceVisitor implements GroupByVisitor {
@@ -27,38 +28,51 @@ public class GroupByReplaceVisitor implements GroupByVisitor {
}
public void visit(GroupByElement groupByElement) {
groupByElement.getGroupByExpressionList();
ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList();
List<Expression> groupByExpressions = groupByExpressionList.getExpressions();
for (int i = 0; i < groupByExpressions.size(); i++) {
Expression expression = groupByExpressions.get(i);
String columnName = expression.toString();
if (expression instanceof Function && Objects.nonNull(
((Function) expression).getParameters().getExpressions().get(0))) {
columnName = ((Function) expression).getParameters().getExpressions().get(0).toString();
}
String replaceColumn = parseVisitorHelper.getReplaceValue(columnName, fieldNameMap,
exactReplace);
String columnName = getColumnName(expression);
String replaceColumn = parseVisitorHelper.getReplaceValue(columnName, fieldNameMap, exactReplace);
if (StringUtils.isNotEmpty(replaceColumn)) {
if (expression instanceof Column) {
groupByExpressions.set(i, new Column(replaceColumn));
}
if (expression instanceof Function) {
try {
Expression element = CCJSqlParserUtil.parseExpression(replaceColumn);
ExpressionList<Expression> expressionList = new ExpressionList<Expression>();
expressionList.add(element);
if (((Function) expression).getParameters().size() > 1) {
((Function) expression).getParameters().stream().skip(1).forEach(e -> {
expressionList.add((Function) e);
});
}
((Function) expression).setParameters(expressionList);
} catch (JSQLParserException e) {
log.error("e", e);
}
replaceExpression(groupByExpressions, i, expression, replaceColumn);
}
}
}
private String getColumnName(Expression expression) {
if (expression instanceof Function) {
Function function = (Function) expression;
if (Objects.nonNull(function.getParameters().getExpressions().get(0))) {
return function.getParameters().getExpressions().get(0).toString();
}
}
return expression.toString();
}
private void replaceExpression(List<Expression> groupByExpressions,
int index,
Expression expression,
String replaceColumn) {
if (expression instanceof Column) {
groupByExpressions.set(index, new Column(replaceColumn));
} else if (expression instanceof Function) {
try {
Expression newExpression = CCJSqlParserUtil.parseExpression(replaceColumn);
ExpressionList<Expression> newExpressionList = new ExpressionList<>();
newExpressionList.add(newExpression);
Function function = (Function) expression;
if (function.getParameters().size() > 1) {
function.getParameters().stream().skip(1).forEach(
e -> newExpressionList.add((Function) e)
);
}
function.setParameters(newExpressionList);
} catch (JSQLParserException e) {
log.error("Error parsing expression: {}", replaceColumn, e);
}
}
}

1
common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java
View File

@@ -71,7 +71,6 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
}
}
}
//selectExpressionItem.getExpression().accept(this);
}
public static Expression replace(Expression expression, Map<String, String> fieldExprMap) {

2
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java
View File

@@ -153,7 +153,7 @@ class SqlReplaceHelperTest {
Assert.assertEquals(
"SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 "
+ "AND 歌手名 LIKE '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = "
+ "AND 歌曲名 LIKE '%邓紫棋%' AND 数据日期 = '2023-08-09' AND 歌曲发布时 = "
+ "'2023-08-01' ORDER BY 播放量 DESC LIMIT 11", replaceSql);
Set<String> fieldNames = new HashSet<>();