From 4a876b97df4fb750cbfa14db28630fdd52a74827 Mon Sep 17 00:00:00 2001 From: mainmain <57514971+mainmainer@users.noreply.github.com> Date: Tue, 25 Jun 2024 11:13:37 +0800 Subject: [PATCH] =?UTF-8?q?(improvement)(headless)=20fix=20SqlReplaceHelpe?= =?UTF-8?q?r=20replaceFields=EF=BC=8Cimprove=20subselect=20and=20supports?= =?UTF-8?q?=20join=20=20(#1210)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - .../common/jsqlparser/SqlReplaceHelper.java | 81 ++++++++++++++++--- .../common/jsqlparser/SqlSelectHelper.java | 61 ++++++++++---- .../headless/server/utils/DictUtils.java | 2 +- .../src/test/resources/application-local.yaml | 2 +- 5 files changed, 118 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 6d80611fb..8d4de8975 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ target/ .vscode/ logs/ log/ -venv/ *.DS_Store *.iml *.bin diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index 96550f864..2653ea50d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.common.jsqlparser; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.StringUtil; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -9,6 +10,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.UnaryOperator; + import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Alias; @@ -38,6 +40,7 @@ import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.FromItem; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; @@ -119,14 +122,14 @@ public class SqlReplaceHelper { } public static String replaceValue(String sql, Map> filedNameToValueMap, - boolean exactReplace) { + boolean exactReplace) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; } - List plainSelectList = new ArrayList<>(); - plainSelectList.add((PlainSelect) selectStatement); - List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); + //List plainSelectList = new ArrayList<>(); + //plainSelectList.add((PlainSelect) selectStatement); + List plainSelects = SqlSelectHelper.getPlainSelect(selectStatement); for (PlainSelect plainSelect : plainSelects) { Expression where = plainSelect.getWhere(); FieldlValueReplaceVisitor visitor = new FieldlValueReplaceVisitor(exactReplace, filedNameToValueMap); @@ -155,6 +158,28 @@ public class SqlReplaceHelper { return selectStatement.toString(); } + public static void getFromSelect(FromItem fromItem, List plainSelectList) { + if (!(fromItem instanceof ParenthesedSelect)) { + return; + } + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) fromItem; + Select select = parenthesedSelect.getSelect(); + if (select instanceof PlainSelect) { + PlainSelect plainSelect = (PlainSelect) select; + plainSelectList.add(plainSelect); + getFromSelect(plainSelect.getFromItem(), plainSelectList); + } else if (select instanceof SetOperationList) { + SetOperationList setOperationList = (SetOperationList) select; + if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { + setOperationList.getSelects().forEach(subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + getFromSelect(subPlainSelect.getFromItem(), plainSelectList); + }); + } + } + } + public static String replaceFields(String sql, Map fieldNameMap) { return replaceFields(sql, fieldNameMap, false); } @@ -164,13 +189,19 @@ public class SqlReplaceHelper { List plainSelectList = SqlSelectHelper.getWithItem(selectStatement); //plainSelectList.add(selectStatement.getPlainSelect()); if (selectStatement instanceof PlainSelect) { - plainSelectList.add((PlainSelect) selectStatement); + PlainSelect plainSelect = (PlainSelect) selectStatement; + plainSelectList.add(plainSelect); + getFromSelect(plainSelect.getFromItem(), plainSelectList); + //plainSelectList.add((PlainSelect) selectStatement); } else if (selectStatement instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) selectStatement; if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { setOperationList.getSelects().forEach(subSelectBody -> { + //PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + //plainSelectList.add(subPlainSelect); PlainSelect subPlainSelect = (PlainSelect) subSelectBody; plainSelectList.add(subPlainSelect); + getFromSelect(subPlainSelect.getFromItem(), plainSelectList); }); } List orderByElements = setOperationList.getOrderByElements(); @@ -190,7 +221,7 @@ public class SqlReplaceHelper { } private static void replaceFieldsInPlainOneSelect(Map fieldNameMap, boolean exactReplace, - PlainSelect plainSelect) { + PlainSelect plainSelect) { //1. replace where fields Expression where = plainSelect.getWhere(); FieldReplaceVisitor visitor = new FieldReplaceVisitor(fieldNameMap, exactReplace); @@ -206,8 +237,19 @@ public class SqlReplaceHelper { if (plainSelect.getFromItem() instanceof ParenthesedSelect) { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); - PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); + Select select = parenthesedSelect.getSelect(); + if (select instanceof PlainSelect) { + PlainSelect subPlainSelect = (PlainSelect) select; + replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); + } else if (select instanceof SetOperationList) { + SetOperationList setOperationList = (SetOperationList) select; + if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { + setOperationList.getSelects().forEach(subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); + }); + } + } } //3. replace oder by fields @@ -238,8 +280,9 @@ public class SqlReplaceHelper { if (!(join.getRightItem() instanceof ParenthesedSelect)) { continue; } + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getRightItem(); List plainSelectList = new ArrayList<>(); - plainSelectList.add((PlainSelect) join.getRightItem()); + plainSelectList.add(parenthesedSelect.getPlainSelect()); List subPlainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect subPlainSelect : subPlainSelects) { replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); @@ -267,7 +310,7 @@ public class SqlReplaceHelper { } public static String replaceFunction(String sql, Map functionMap, - Map functionCall) { + Map functionCall) { Select selectStatement = SqlSelectHelper.getSelect(sql); if (!(selectStatement instanceof PlainSelect)) { return sql; @@ -282,7 +325,7 @@ public class SqlReplaceHelper { } private static void replaceFunction(Map functionMap, Map functionCall, - PlainSelect selectBody) { + PlainSelect selectBody) { PlainSelect plainSelect = selectBody; //1. replace where dataDiff function Expression where = plainSelect.getWhere(); @@ -360,7 +403,7 @@ public class SqlReplaceHelper { } private static void replaceOrderByFunction(Map functionMap, - List orderByElementList) { + List orderByElementList) { if (Objects.isNull(orderByElementList)) { return; } @@ -435,6 +478,16 @@ public class SqlReplaceHelper { PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); replaceSingleTable(subPlainSelect, tableName); } + List joinList = plainSelect.getJoins(); + if (CollectionUtils.isEmpty(joinList)) { + return; + } + for (Join join : joinList) { + if (join.getFromItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getFromItem(); + replaceSingleTable(parenthesedSelect.getPlainSelect(), tableName); + } + } } public static void replaceSingleTable(PlainSelect plainSelect, String tableName) { @@ -648,6 +701,10 @@ public class SqlReplaceHelper { public static String dealAliasToOrderBy(String querySql) { Select selectStatement = SqlSelectHelper.getSelect(querySql); List plainSelectList = new ArrayList<>(); + //List withPlainSelectList = SqlSelectHelper.getWithItem(selectStatement); + //if (!CollectionUtils.isEmpty(withPlainSelectList)) { + // plainSelectList.addAll(withPlainSelectList); + //} if (selectStatement instanceof PlainSelect) { plainSelectList.add((PlainSelect) selectStatement); } else if (selectStatement instanceof SetOperationList) { diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java index 4f9818bae..90db3c064 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java @@ -116,8 +116,7 @@ public class SqlSelectHelper { return result; } - public static List getPlainSelect(String sql) { - Select selectStatement = getSelect(sql); + public static List getPlainSelect(Select selectStatement) { if (selectStatement == null) { return null; } @@ -139,6 +138,11 @@ public class SqlSelectHelper { return plainSelectList; } + public static List getPlainSelect(String sql) { + Select selectStatement = getSelect(sql); + return getPlainSelect(selectStatement); + } + public static Boolean hasSubSelect(String sql) { Select selectStatement = getSelect(sql); if (selectStatement == null) { @@ -162,6 +166,16 @@ public class SqlSelectHelper { Select subSelect = parenthesedSelect.getSelect(); getSubPlainSelect(subSelect, plainSelectList); } + List joinList = plainSelect.getJoins(); + if (CollectionUtils.isEmpty(joinList)) { + return; + } + for (Join join : joinList) { + if (join.getRightItem() instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) join.getRightItem(); + plainSelectList.add(parenthesedSelect.getPlainSelect()); + } + } } if (select instanceof SetOperationList) { SetOperationList setOperationList = (SetOperationList) select; @@ -313,10 +327,28 @@ public class SqlSelectHelper { } if (plainSelect.getFromItem() instanceof ParenthesedSelect) { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); - PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - Expression subWhere = subPlainSelect.getWhere(); - if (Objects.nonNull(subWhere)) { - subWhere.accept(new FieldAndValueAcquireVisitor(result)); + Select subSelect = parenthesedSelect.getSelect(); + if (subSelect instanceof PlainSelect) { + PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); + Expression subWhere = subPlainSelect.getWhere(); + if (Objects.nonNull(subWhere)) { + subWhere.accept(new FieldAndValueAcquireVisitor(result)); + } + } else if (subSelect instanceof ParenthesedSelect) { + ParenthesedSelect subParenthesedSelect = (ParenthesedSelect) subSelect; + Expression subWhere = subParenthesedSelect.getPlainSelect().getWhere(); + if (Objects.nonNull(subWhere)) { + subWhere.accept(new FieldAndValueAcquireVisitor(result)); + } + } else if (subSelect instanceof SetOperationList) { + SetOperationList setOperationList = (SetOperationList) subSelect; + List