topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](chat) remove duplicates from multiple SQL identified by LLM. (#391)

Browse Source
This commit is contained in:
lexluo09
2023-11-16 09:59:54 +08:00
committed by GitHub
parent 8688c8c2b3
commit 5b3a9ffba8
16 changed files with 348 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java
View File

@@ -28,38 +28,38 @@ import org.apache.commons.collections.CollectionUtils;
public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
private Set<FilterExpression> filterExpressions;
private Set<FieldExpression> fieldExpressions;
public FieldAndValueAcquireVisitor(Set<FilterExpression> filterExpressions) {
this.filterExpressions = filterExpressions;
public FieldAndValueAcquireVisitor(Set<FieldExpression> fieldExpressions) {
this.fieldExpressions = fieldExpressions;
}
public void visit(LikeExpression expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
FilterExpression filterExpression = new FilterExpression();
FieldExpression fieldExpression = new FieldExpression();
String columnName = null;
if (leftExpression instanceof Column) {
Column column = (Column) leftExpression;
columnName = column.getColumnName();
filterExpression.setFieldName(columnName);
fieldExpression.setFieldName(columnName);
}
filterExpression.setFieldValue(getFieldValue(rightExpression));
filterExpression.setOperator(expr.getStringExpression());
filterExpressions.add(filterExpression);
fieldExpression.setFieldValue(getFieldValue(rightExpression));
fieldExpression.setOperator(expr.getStringExpression());
fieldExpressions.add(fieldExpression);
}
public void visit(InExpression expr) {
FilterExpression filterExpression = new FilterExpression();
FieldExpression fieldExpression = new FieldExpression();
Expression leftExpression = expr.getLeftExpression();
if (!(leftExpression instanceof Column)) {
return;
}
filterExpression.setFieldName(((Column) leftExpression).getColumnName());
filterExpression.setOperator(JsqlConstants.IN);
fieldExpression.setFieldName(((Column) leftExpression).getColumnName());
fieldExpression.setOperator(JsqlConstants.IN);
ItemsList rightItemsList = expr.getRightItemsList();
filterExpression.setFieldValue(rightItemsList);
fieldExpression.setFieldValue(rightItemsList);
List<Object> result = new ArrayList<>();
if (rightItemsList instanceof ExpressionList) {
ExpressionList rightExpressionList = (ExpressionList) rightItemsList;
@@ -70,78 +70,78 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
}
}
}
filterExpression.setFieldValue(result);
filterExpressions.add(filterExpression);
fieldExpression.setFieldValue(result);
fieldExpressions.add(fieldExpression);
}
@Override
public void visit(MinorThan expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
FieldExpression fieldExpression = getFilterExpression(expr);
fieldExpressions.add(fieldExpression);
}
@Override
public void visit(EqualsTo expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
FieldExpression fieldExpression = getFilterExpression(expr);
fieldExpressions.add(fieldExpression);
}
@Override
public void visit(MinorThanEquals expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
FieldExpression fieldExpression = getFilterExpression(expr);
fieldExpressions.add(fieldExpression);
}
@Override
public void visit(GreaterThan expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
FieldExpression fieldExpression = getFilterExpression(expr);
fieldExpressions.add(fieldExpression);
}
@Override
public void visit(GreaterThanEquals expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
FieldExpression fieldExpression = getFilterExpression(expr);
fieldExpressions.add(fieldExpression);
}
private FilterExpression getFilterExpression(ComparisonOperator expr) {
private FieldExpression getFilterExpression(ComparisonOperator expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
FilterExpression filterExpression = new FilterExpression();
FieldExpression fieldExpression = new FieldExpression();
String columnName = null;
if (leftExpression instanceof Column) {
Column column = (Column) leftExpression;
columnName = column.getColumnName();
filterExpression.setFieldName(columnName);
fieldExpression.setFieldName(columnName);
}
if (leftExpression instanceof Function) {
Function leftExpressionFunction = (Function) leftExpression;
Column field = getColumn(leftExpressionFunction);
if (Objects.isNull(field)) {
return filterExpression;
return fieldExpression;
}
String functionName = leftExpressionFunction.getName().toUpperCase();
filterExpression.setFieldName(field.getColumnName());
filterExpression.setFunction(functionName);
filterExpression.setOperator(expr.getStringExpression());
fieldExpression.setFieldName(field.getColumnName());
fieldExpression.setFunction(functionName);
fieldExpression.setOperator(expr.getStringExpression());
//deal with DAY/WEEK function
List<DatePeriodEnum> collect = Arrays.stream(DatePeriodEnum.values()).collect(Collectors.toList());
DatePeriodEnum periodEnum = DatePeriodEnum.get(functionName);
if (Objects.nonNull(periodEnum) && collect.contains(periodEnum)) {
filterExpression.setFieldValue(getFieldValue(rightExpression) + periodEnum.getChName());
return filterExpression;
fieldExpression.setFieldValue(getFieldValue(rightExpression) + periodEnum.getChName());
return fieldExpression;
} else {
//deal with aggregate function
filterExpression.setFieldValue(getFieldValue(rightExpression));
return filterExpression;
fieldExpression.setFieldValue(getFieldValue(rightExpression));
return fieldExpression;
}
}
filterExpression.setFieldValue(getFieldValue(rightExpression));
filterExpression.setOperator(expr.getStringExpression());
return filterExpression;
fieldExpression.setFieldValue(getFieldValue(rightExpression));
fieldExpression.setOperator(expr.getStringExpression());
return fieldExpression;
}
private Column getColumn(Function leftExpressionFunction) {

2
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FilterExpression.java → common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldExpression.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.common.util.jsqlparser;
import lombok.Data;
@Data
public class FilterExpression {
public class FieldExpression {
private String operator;

16
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.List;
import java.util.Set;
import net.sf.jsqlparser.expression.Expression;
@@ -10,27 +11,34 @@ import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter;
public class OrderByAcquireVisitor extends OrderByVisitorAdapter {
private Set<String> fields;
private Set<FieldExpression> fields;
public OrderByAcquireVisitor(Set<String> fields) {
public OrderByAcquireVisitor(Set<FieldExpression> fields) {
this.fields = fields;
}
@Override
public void visit(OrderByElement orderBy) {
Expression expression = orderBy.getExpression();
FieldExpression fieldExpression = new FieldExpression();
if (expression instanceof Column) {
fields.add(((Column) expression).getColumnName());
fieldExpression.setFieldName(((Column) expression).getColumnName());
}
if (expression instanceof Function) {
Function function = (Function) expression;
List<Expression> expressions = function.getParameters().getExpressions();
for (Expression column : expressions) {
if (column instanceof Column) {
fields.add(((Column) column).getColumnName());
fieldExpression.setFieldName(((Column) column).getColumnName());
}
}
}
String operator = Constants.ASC_UPPER;
if (!orderBy.isAsc()) {
operator = Constants.DESC_UPPER;
}
fieldExpression.setOperator(operator);
fields.add(fieldExpression);
super.visit(orderBy);
}
}

16
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByExpression.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.common.util.jsqlparser;
import lombok.Data;
@Data
public class OrderByExpression {
private String operator;
private String fieldName;
private Object fieldValue;
private String function;
}

67
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelper.java Normal file
View File

@@ -0,0 +1,67 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
/**
* Sql Parser equal Helper
*/
@Slf4j
public class SqlParserEqualHelper {
/**
* determine if two SQL statements are equal.
*
* @param thisSql
* @param otherSql
* @return
*/
public static boolean equals(String thisSql, String otherSql) {
//1. select fields
List<String> thisSelectFields = SqlParserSelectHelper.getSelectFields(thisSql);
List<String> otherSelectFields = SqlParserSelectHelper.getSelectFields(otherSql);
if (!CollectionUtils.isEqualCollection(thisSelectFields, otherSelectFields)) {
return false;
}
//2. all fields
List<String> thisAllFields = SqlParserSelectHelper.getAllFields(thisSql);
List<String> otherAllFields = SqlParserSelectHelper.getAllFields(otherSql);
if (!CollectionUtils.isEqualCollection(thisAllFields, otherAllFields)) {
return false;
}
//3. where
List<FieldExpression> thisFieldExpressions = SqlParserSelectHelper.getFilterExpression(thisSql);
List<FieldExpression> otherFieldExpressions = SqlParserSelectHelper.getFilterExpression(otherSql);
if (!CollectionUtils.isEqualCollection(thisFieldExpressions, otherFieldExpressions)) {
return false;
}
//4. tableName
if (!SqlParserSelectHelper.getDbTableName(thisSql)
.equalsIgnoreCase(SqlParserSelectHelper.getDbTableName(otherSql))) {
return false;
}
//5. having
List<FieldExpression> thisHavingExpressions = SqlParserSelectHelper.getHavingExpressions(thisSql);
List<FieldExpression> otherHavingExpressions = SqlParserSelectHelper.getHavingExpressions(otherSql);
if (!CollectionUtils.isEqualCollection(thisHavingExpressions, otherHavingExpressions)) {
return false;
}
//6. orderBy
List<FieldExpression> thisOrderByExpressions = SqlParserSelectHelper.getOrderByExpressions(thisSql);
List<FieldExpression> otherOrderByExpressions = SqlParserSelectHelper.getOrderByExpressions(otherSql);
if (!CollectionUtils.isEqualCollection(thisOrderByExpressions, otherOrderByExpressions)) {
return false;
}
return true;
}
}

33
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java
View File

@@ -5,6 +5,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -40,12 +41,12 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class SqlParserSelectHelper {
public static List<FilterExpression> getFilterExpression(String sql) {
public static List<FieldExpression> getFilterExpression(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<FilterExpression> result = new HashSet<>();
Set<FieldExpression> result = new HashSet<>();
Expression where = plainSelect.getWhere();
if (Objects.nonNull(where)) {
where.accept(new FieldAndValueAcquireVisitor(result));
@@ -208,12 +209,12 @@ public class SqlParserSelectHelper {
return null;
}
public static List<FilterExpression> getWhereExpressions(String sql) {
public static List<FieldExpression> getWhereExpressions(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<FilterExpression> result = new HashSet<>();
Set<FieldExpression> result = new HashSet<>();
Expression where = plainSelect.getWhere();
if (Objects.nonNull(where)) {
where.accept(new FieldAndValueAcquireVisitor(result));
@@ -221,12 +222,12 @@ public class SqlParserSelectHelper {
return new ArrayList<>(result);
}
public static List<FilterExpression> getHavingExpressions(String sql) {
public static List<FieldExpression> getHavingExpressions(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<FilterExpression> result = new HashSet<>();
Set<FieldExpression> result = new HashSet<>();
Expression having = plainSelect.getHaving();
if (Objects.nonNull(having)) {
having.accept(new FieldAndValueAcquireVisitor(result));
@@ -244,13 +245,31 @@ public class SqlParserSelectHelper {
return new ArrayList<>(result);
}
private static void getOrderByFields(PlainSelect plainSelect, Set<String> result) {
private static Set<FieldExpression> getOrderByFields(PlainSelect plainSelect) {
Set<FieldExpression> result = new HashSet<>();
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (!CollectionUtils.isEmpty(orderByElements)) {
for (OrderByElement orderByElement : orderByElements) {
orderByElement.accept(new OrderByAcquireVisitor(result));
}
}
return result;
}
private static void getOrderByFields(PlainSelect plainSelect, Set<String> result) {
Set<FieldExpression> orderByFieldExpressions = getOrderByFields(plainSelect);
Set<String> collect = orderByFieldExpressions.stream()
.map(fieldExpression -> fieldExpression.getFieldName())
.collect(Collectors.toSet());
result.addAll(collect);
}
public static List<FieldExpression> getOrderByExpressions(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
return new ArrayList<>(getOrderByFields(plainSelect));
}
public static List<String> getGroupByFields(String sql) {