(improvement)(common) Support deleting 'is null' and 'is not null' expressions in the WHERE clause (#1670)

common/src/main/java/com/tencent/supersonic/common/jsqlparser/CustomExpressionDeParser.java

@@ -0,0 +1,83 @@
+package com.tencent.supersonic.common.jsqlparser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
+import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
+import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
+
+import java.util.Set;
+
+public class CustomExpressionDeParser extends ExpressionDeParser {
+
+    private Set<String> removeFieldNames;
+    private boolean dealNull;
+    private boolean dealNotNull;
+
+    public CustomExpressionDeParser(
+            Set<String> removeFieldNames, boolean dealNull, boolean dealNotNull) {
+        this.removeFieldNames = removeFieldNames;
+        this.dealNull = dealNull;
+        this.dealNotNull = dealNotNull;
+    }
+
+    @Override
+    public void visit(AndExpression andExpression) {
+        processBinaryExpression(andExpression, " AND ");
+    }
+
+    @Override
+    public void visit(OrExpression orExpression) {
+        processBinaryExpression(orExpression, " OR ");
+    }
+
+    @Override
+    public void visit(IsNullExpression isNullExpression) {
+        if (shouldSkip(isNullExpression)) {
+            // Skip this expression
+        } else {
+            super.visit(isNullExpression);
+        }
+    }
+
+    private void processBinaryExpression(Expression binaryExpression, String operator) {
+        Expression leftExpression = ((AndExpression) binaryExpression).getLeftExpression();
+        Expression rightExpression = ((AndExpression) binaryExpression).getRightExpression();
+
+        boolean leftIsNull =
+                leftExpression instanceof IsNullExpression
+                        && shouldSkip((IsNullExpression) leftExpression);
+        boolean rightIsNull =
+                rightExpression instanceof IsNullExpression
+                        && shouldSkip((IsNullExpression) rightExpression);
+
+        if (leftIsNull && rightIsNull) {
+            // Skip both expressions
+        } else if (leftIsNull) {
+            rightExpression.accept(this);
+        } else if (rightIsNull) {
+            leftExpression.accept(this);
+        } else {
+            leftExpression.accept(this);
+            buffer.append(operator);
+            rightExpression.accept(this);
+        }
+    }
+
+    private boolean shouldSkip(IsNullExpression isNullExpression) {
+        if (isNullExpression.getLeftExpression() instanceof Column) {
+            Column column = (Column) isNullExpression.getLeftExpression();
+            String columnName = column.getColumnName();
+            // Add your target column names here
+            if (removeFieldNames.contains(columnName)) {
+                if (isNullExpression.isNot() && dealNotNull) {
+                    return true;
+                } else if (!isNullExpression.isNot() && dealNull) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+}

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java

@@ -27,6 +27,8 @@ import net.sf.jsqlparser.statement.select.Select;
 import net.sf.jsqlparser.statement.select.SelectItem;
 import net.sf.jsqlparser.statement.select.SelectItemVisitorAdapter;
 import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
+import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
+import net.sf.jsqlparser.util.deparser.SelectDeParser;
 import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
@@ -382,6 +384,33 @@ public class SqlRemoveHelper {
         }
     }
 
+    public static String removeIsNullInWhere(String sql, Set<String> removeFieldNames) {
+        return removeIsNullOrNotNullInWhere(true, false, sql, removeFieldNames);
+    }
+
+    public static String removeNotNullInWhere(String sql, Set<String> removeFieldNames) {
+        return removeIsNullOrNotNullInWhere(false, true, sql, removeFieldNames);
+    }
+
+    public static String removeIsNullOrNotNullInWhere(
+            boolean dealNull, boolean dealNotNull, String sql, Set<String> removeFieldNames) {
+        Select selectStatement = SqlSelectHelper.getSelect(sql);
+        if (!(selectStatement instanceof PlainSelect)) {
+            return sql;
+        }
+        // Create a custom ExpressionDeParser to remove specific IS NULL and IS NOT NULL conditions
+        ExpressionDeParser expressionDeParser =
+                new CustomExpressionDeParser(removeFieldNames, dealNull, dealNotNull);
+
+        StringBuilder buffer = new StringBuilder();
+        SelectDeParser selectDeParser = new SelectDeParser(expressionDeParser, buffer);
+        expressionDeParser.setSelectVisitor(selectDeParser);
+        expressionDeParser.setBuffer(buffer);
+
+        selectStatement.accept(selectDeParser);
+        return buffer.toString();
+    }
+
     private static boolean isInvalidSelect(Select selectStatement) {
         return Objects.isNull(selectStatement) || !(selectStatement instanceof PlainSelect);
     }

common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java

@@ -171,4 +171,34 @@ class SqlRemoveHelperTest {
                 "SELECT 数据日期 FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'",
                 replaceSql);
     }
+
+    @Test
+    void testRemoveIsNullInWhere() {
+        String sql =
+                "select 数据日期 from 歌曲库 where 歌曲名 is null and 数据日期 = '2023-08-09' and "
+                        + "歌曲发布时间 = '2023-08-01' group by 数据日期";
+
+        Set<String> removeFieldNames = new HashSet<>();
+        removeFieldNames.add("歌曲名");
+        String replaceSql = SqlRemoveHelper.removeIsNullInWhere(sql, removeFieldNames);
+
+        Assert.assertEquals(
+                "SELECT 数据日期 FROM 歌曲库 WHERE 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01' GROUP BY 数据日期",
+                replaceSql);
+    }
+
+    @Test
+    void testRemoveIsNotNullInWhere() {
+        String sql =
+                "select 数据日期 from 歌曲库 where 歌曲名 is not null and 数据日期 = '2023-08-09' and "
+                        + "歌曲发布时间 = '2023-08-01' group by 数据日期";
+
+        Set<String> removeFieldNames = new HashSet<>();
+        removeFieldNames.add("歌曲名");
+        String replaceSql = SqlRemoveHelper.removeNotNullInWhere(sql, removeFieldNames);
+
+        Assert.assertEquals(
+                "SELECT 数据日期 FROM 歌曲库 WHERE 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01' GROUP BY 数据日期",
+                replaceSql);
+    }
 }