From cadb743eda8f9cd9ff4fa3bcf77865ede1c741e4 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:20:56 +0800 Subject: [PATCH 1/2] (improvement)[build] Add the spotless.skip configuration to resolve the error reported by Spotless during the build. (#1762) --- assembly/bin/supersonic-build.bat | 2 +- assembly/bin/supersonic-build.sh | 2 +- pom.xml | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/assembly/bin/supersonic-build.bat b/assembly/bin/supersonic-build.bat index 725b8ecb3..b554cbd34 100644 --- a/assembly/bin/supersonic-build.bat +++ b/assembly/bin/supersonic-build.bat @@ -40,7 +40,7 @@ if "%service%"=="webapp" ( :buildJavaService set "model_name=%service%" echo "starting building supersonic-%model_name% service" - call mvn -f %projectDir% clean package -DskipTests + call mvn -f %projectDir% clean package -DskipTests -Dspotless.skip=true IF ERRORLEVEL 1 ( ECHO Failed to build backend Java modules. EXIT /B 1 diff --git a/assembly/bin/supersonic-build.sh b/assembly/bin/supersonic-build.sh index b51522fde..8a92373d3 100755 --- a/assembly/bin/supersonic-build.sh +++ b/assembly/bin/supersonic-build.sh @@ -15,7 +15,7 @@ fi function buildJavaService { model_name=$1 echo "starting building supersonic-${model_name} service" - mvn -f $projectDir clean package -DskipTests + mvn -f $projectDir clean package -DskipTests -Dspotless.skip=true if [ $? -ne 0 ]; then echo "Failed to build backend Java modules." exit 1 diff --git a/pom.xml b/pom.xml index 8879c414b..bccb7641c 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ 15.0.2 2.10.1 2.27.1 + false 4.2.1 @@ -244,6 +245,7 @@ spotless-maven-plugin ${spotless.version} + ${spotless.skip} java-formatter.xml @@ -260,6 +262,9 @@ apply + + ${spotless.skip} + From 1215efbdcee4406516b4ed29e505713fe4639ae0 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:14:18 +0800 Subject: [PATCH 2/2] (improvement)[headless] Optimize table name replacement in complex SQL (#1763) --- .../jsqlparser/FieldValueReplaceVisitor.java | 3 + .../common/jsqlparser/SqlReplaceHelper.java | 129 +++++++++--------- .../jsqlparser/TableNameReplaceVisitor.java | 9 +- .../jsqlparser/SqlReplaceHelperTest.java | 12 ++ 4 files changed, 85 insertions(+), 68 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java index 182da9d00..ad524cc9a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/FieldValueReplaceVisitor.java @@ -64,6 +64,9 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter { } Column column = (Column) inExpression.getLeftExpression(); Map valueMap = filedNameToValueMap.get(column.getColumnName()); + if (!(inExpression.getRightExpression() instanceof ExpressionList)) { + return; + } ExpressionList rightItemsList = (ExpressionList) inExpression.getRightExpression(); List expressions = rightItemsList.getExpressions(); List values = new ArrayList<>(); diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index 3f69ad0e9..f1c3a54f6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -36,7 +36,9 @@ import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -183,7 +185,6 @@ public class SqlReplaceHelper { } - private static void replaceFieldsInPlainOneSelect(Map fieldNameMap, boolean exactReplace, PlainSelect plainSelect) { // 1. replace where fields @@ -385,95 +386,89 @@ public class SqlReplaceHelper { if (StringUtils.isEmpty(tableName)) { return sql; } + List withNameList = SqlSelectHelper.getWithName(sql); Select selectStatement = SqlSelectHelper.getSelect(sql); List plainSelectList = SqlSelectHelper.getWithItem(selectStatement); + if (!CollectionUtils.isEmpty(plainSelectList)) { - List withNameList = SqlSelectHelper.getWithName(sql); - plainSelectList.stream().forEach(plainSelect -> { - if (plainSelect.getFromItem() instanceof Table) { - Table table = (Table) plainSelect.getFromItem(); - if (!withNameList.contains(table.getName())) { - replaceSingleTable(plainSelect, tableName); - } - } - if (plainSelect.getFromItem() instanceof ParenthesedSelect) { - ParenthesedSelect parenthesedSelect = - (ParenthesedSelect) plainSelect.getFromItem(); - PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - Table table = (Table) subPlainSelect.getFromItem(); - if (!withNameList.contains(table.getName())) { - replaceSingleTable(subPlainSelect, tableName); - } - } - }); - return selectStatement.toString(); + plainSelectList.forEach( + plainSelect -> processPlainSelect(plainSelect, tableName, withNameList)); } if (selectStatement instanceof PlainSelect) { - PlainSelect plainSelect = (PlainSelect) selectStatement; - replaceSingleTable(plainSelect, tableName); - replaceSubTable(plainSelect, tableName); + processPlainSelect((PlainSelect) selectStatement, tableName, withNameList); } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { - setOperationList.getSelects().forEach(subSelectBody -> { - PlainSelect subPlainSelect = (PlainSelect) subSelectBody; - replaceSingleTable(subPlainSelect, tableName); - replaceSubTable(subPlainSelect, tableName); - }); + setOperationList.getSelects() + .forEach(subSelectBody -> processPlainSelect((PlainSelect) subSelectBody, + tableName, withNameList)); } } return selectStatement.toString(); } - public static void replaceSubTable(PlainSelect plainSelect, String tableName) { - if (plainSelect.getFromItem() instanceof ParenthesedSelect) { + private static void processPlainSelect(PlainSelect plainSelect, String tableName, + List withNameList) { + if (plainSelect.getFromItem() instanceof Table) { + replaceSingleTable(plainSelect, tableName, withNameList); + } else if (plainSelect.getFromItem() instanceof ParenthesedSelect) { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - replaceSingleTable(subPlainSelect, tableName); + replaceSingleTable(subPlainSelect, tableName, withNameList); } - List joinList = plainSelect.getJoins(); - if (CollectionUtils.isEmpty(joinList)) { - return; - } - for (Join join : joinList) { - if (join.getFromItem() instanceof ParenthesedSelect) { - ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getFromItem(); - replaceSingleTable(parenthesedSelect.getPlainSelect(), tableName); + replaceSubTable(plainSelect, tableName, withNameList); + } + + public static void replaceSingleTable(PlainSelect plainSelect, String tableName, + List withNameList) { + List plainSelects = + SqlSelectHelper.getPlainSelects(Collections.singletonList(plainSelect)); + plainSelects.forEach(painSelect -> { + painSelect.accept(new SelectVisitorAdapter() { + @Override + public void visit(PlainSelect plainSelect) { + plainSelect.getFromItem().accept( + new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList))); + } + }); + replaceJoins(painSelect, tableName, withNameList); + }); + } + + private static void replaceJoins(PlainSelect plainSelect, String tableName, + List withNameList) { + List joins = plainSelect.getJoins(); + if (!CollectionUtils.isEmpty(joins)) { + for (Join join : joins) { + if (join.getRightItem() instanceof ParenthesedFromItem) { + List subPlainSelects = SqlSelectHelper.getPlainSelects( + Collections.singletonList((PlainSelect) join.getRightItem())); + subPlainSelects.forEach(subPlainSelect -> subPlainSelect.getFromItem().accept( + new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList)))); + } else if (join.getRightItem() instanceof Table) { + Table table = (Table) join.getRightItem(); + table.setName(tableName); + } } } } - public static void replaceSingleTable(PlainSelect plainSelect, String tableName) { - // replace table name - List plainSelects = new ArrayList<>(); - plainSelects.add(plainSelect); - List painSelects = SqlSelectHelper.getPlainSelects(plainSelects); - for (PlainSelect painSelect : painSelects) { - painSelect.accept(new SelectVisitorAdapter() { - @Override - public void visit(PlainSelect plainSelect) { - plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName)); + public static void replaceSubTable(PlainSelect plainSelect, String tableName, + List withNameList) { + if (plainSelect.getFromItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); + replaceSingleTable(parenthesedSelect.getPlainSelect(), tableName, withNameList); + } + + List joinList = plainSelect.getJoins(); + if (!CollectionUtils.isEmpty(joinList)) { + joinList.forEach(join -> { + if (join.getFromItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getFromItem(); + replaceSingleTable(parenthesedSelect.getPlainSelect(), tableName, withNameList); } }); - List joins = painSelect.getJoins(); - if (!CollectionUtils.isEmpty(joins)) { - for (Join join : joins) { - if (join.getRightItem() instanceof ParenthesedFromItem) { - List plainSelectList = new ArrayList<>(); - plainSelectList.add((PlainSelect) join.getRightItem()); - List subPlainSelects = - SqlSelectHelper.getPlainSelects(plainSelectList); - for (PlainSelect subPlainSelect : subPlainSelects) { - subPlainSelect.getFromItem() - .accept(new TableNameReplaceVisitor(tableName)); - } - } else if (join.getRightItem() instanceof Table) { - Table table = (Table) join.getRightItem(); - table.setName(tableName); - } - } - } } } diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java index 89a67bfd8..148ab8c8b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/TableNameReplaceVisitor.java @@ -3,16 +3,23 @@ package com.tencent.supersonic.common.jsqlparser; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.select.FromItemVisitorAdapter; +import java.util.Set; + public class TableNameReplaceVisitor extends FromItemVisitorAdapter { + private Set notReplaceTables; private String tableName; - public TableNameReplaceVisitor(String tableName) { + public TableNameReplaceVisitor(String tableName, Set notReplaceTables) { this.tableName = tableName; + this.notReplaceTables = notReplaceTables; } @Override public void visit(Table table) { + if (notReplaceTables.contains(table.getName())) { + return; + } table.setName(tableName); } } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java index 19af21a0c..a1799cdbe 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java @@ -402,6 +402,18 @@ class SqlReplaceHelperTest { Assert.assertEquals("SELECT 歌曲名称, sum(评分) FROM cspider WHERE (1 < 2) AND 数据日期 = " + "'2023-10-15' GROUP BY 歌曲名称 HAVING sum(评分) < (SELECT min(评分) " + "FROM cspider WHERE 语种 = '英文')", replaceSql); + + sql = "WITH _部门访问次数_ AS ( SELECT 部门, SUM(访问次数) AS _总访问次数_ FROM 超音数数据集 WHERE 数据日期 >= '2024-07-11'" + + " AND 数据日期 <= '2024-10-09' GROUP BY 部门 HAVING SUM(访问次数) > 100 ) SELECT 用户, SUM(访问次数) " + + "AS _访问次数汇总_ FROM 超音数数据集 WHERE 部门 IN ( SELECT 部门 FROM _部门访问次数_ ) AND 数据日期 >= '2024-07-11' " + + "AND 数据日期 <= '2024-10-09' GROUP BY 用户"; + + replaceSql = SqlReplaceHelper.replaceTable(sql, "t_1"); + + Assert.assertEquals("WITH _部门访问次数_ AS (SELECT 部门, SUM(访问次数) AS _总访问次数_ FROM t_1 " + + "WHERE 数据日期 >= '2024-07-11' AND 数据日期 <= '2024-10-09' GROUP BY 部门 HAVING SUM(访问次数) > 100) " + + "SELECT 用户, SUM(访问次数) AS _访问次数汇总_ FROM t_1 WHERE 部门 IN (SELECT 部门 FROM _部门访问次数_) " + + "AND 数据日期 >= '2024-07-11' AND 数据日期 <= '2024-10-09' GROUP BY 用户", replaceSql); } @Test