diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/CustomExpressionDeParser.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/CustomExpressionDeParser.java new file mode 100644 index 000000000..d7b61118e --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/CustomExpressionDeParser.java @@ -0,0 +1,83 @@ +package com.tencent.supersonic.common.jsqlparser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.conditional.AndExpression; +import net.sf.jsqlparser.expression.operators.conditional.OrExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; + +import java.util.Set; + +public class CustomExpressionDeParser extends ExpressionDeParser { + + private Set removeFieldNames; + private boolean dealNull; + private boolean dealNotNull; + + public CustomExpressionDeParser( + Set removeFieldNames, boolean dealNull, boolean dealNotNull) { + this.removeFieldNames = removeFieldNames; + this.dealNull = dealNull; + this.dealNotNull = dealNotNull; + } + + @Override + public void visit(AndExpression andExpression) { + processBinaryExpression(andExpression, " AND "); + } + + @Override + public void visit(OrExpression orExpression) { + processBinaryExpression(orExpression, " OR "); + } + + @Override + public void visit(IsNullExpression isNullExpression) { + if (shouldSkip(isNullExpression)) { + // Skip this expression + } else { + super.visit(isNullExpression); + } + } + + private void processBinaryExpression(Expression binaryExpression, String operator) { + Expression leftExpression = ((AndExpression) binaryExpression).getLeftExpression(); + Expression rightExpression = ((AndExpression) binaryExpression).getRightExpression(); + + boolean leftIsNull = + leftExpression instanceof IsNullExpression + && shouldSkip((IsNullExpression) leftExpression); + boolean rightIsNull = + rightExpression instanceof IsNullExpression + && shouldSkip((IsNullExpression) rightExpression); + + if (leftIsNull && rightIsNull) { + // Skip both expressions + } else if (leftIsNull) { + rightExpression.accept(this); + } else if (rightIsNull) { + leftExpression.accept(this); + } else { + leftExpression.accept(this); + buffer.append(operator); + rightExpression.accept(this); + } + } + + private boolean shouldSkip(IsNullExpression isNullExpression) { + if (isNullExpression.getLeftExpression() instanceof Column) { + Column column = (Column) isNullExpression.getLeftExpression(); + String columnName = column.getColumnName(); + // Add your target column names here + if (removeFieldNames.contains(columnName)) { + if (isNullExpression.isNot() && dealNotNull) { + return true; + } else if (!isNullExpression.isNot() && dealNull) { + return true; + } + } + } + return false; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java index ce5d63df3..6d977ee5f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java @@ -27,6 +27,8 @@ import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.SelectItemVisitorAdapter; import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -382,6 +384,33 @@ public class SqlRemoveHelper { } } + public static String removeIsNullInWhere(String sql, Set removeFieldNames) { + return removeIsNullOrNotNullInWhere(true, false, sql, removeFieldNames); + } + + public static String removeNotNullInWhere(String sql, Set removeFieldNames) { + return removeIsNullOrNotNullInWhere(false, true, sql, removeFieldNames); + } + + public static String removeIsNullOrNotNullInWhere( + boolean dealNull, boolean dealNotNull, String sql, Set removeFieldNames) { + Select selectStatement = SqlSelectHelper.getSelect(sql); + if (!(selectStatement instanceof PlainSelect)) { + return sql; + } + // Create a custom ExpressionDeParser to remove specific IS NULL and IS NOT NULL conditions + ExpressionDeParser expressionDeParser = + new CustomExpressionDeParser(removeFieldNames, dealNull, dealNotNull); + + StringBuilder buffer = new StringBuilder(); + SelectDeParser selectDeParser = new SelectDeParser(expressionDeParser, buffer); + expressionDeParser.setSelectVisitor(selectDeParser); + expressionDeParser.setBuffer(buffer); + + selectStatement.accept(selectDeParser); + return buffer.toString(); + } + private static boolean isInvalidSelect(Select selectStatement) { return Objects.isNull(selectStatement) || !(selectStatement instanceof PlainSelect); } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java index 700684a1b..cfa9568c6 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java @@ -171,4 +171,34 @@ class SqlRemoveHelperTest { "SELECT 数据日期 FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'", replaceSql); } + + @Test + void testRemoveIsNullInWhere() { + String sql = + "select 数据日期 from 歌曲库 where 歌曲名 is null and 数据日期 = '2023-08-09' and " + + "歌曲发布时间 = '2023-08-01' group by 数据日期"; + + Set removeFieldNames = new HashSet<>(); + removeFieldNames.add("歌曲名"); + String replaceSql = SqlRemoveHelper.removeIsNullInWhere(sql, removeFieldNames); + + Assert.assertEquals( + "SELECT 数据日期 FROM 歌曲库 WHERE 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01' GROUP BY 数据日期", + replaceSql); + } + + @Test + void testRemoveIsNotNullInWhere() { + String sql = + "select 数据日期 from 歌曲库 where 歌曲名 is not null and 数据日期 = '2023-08-09' and " + + "歌曲发布时间 = '2023-08-01' group by 数据日期"; + + Set removeFieldNames = new HashSet<>(); + removeFieldNames.add("歌曲名"); + String replaceSql = SqlRemoveHelper.removeNotNullInWhere(sql, removeFieldNames); + + Assert.assertEquals( + "SELECT 数据日期 FROM 歌曲库 WHERE 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01' GROUP BY 数据日期", + replaceSql); + } }