diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index e9fc3dbcd..6604f2487 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -20,17 +20,7 @@ import net.sf.jsqlparser.expression.operators.relational.NotEqualsTo; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; -import net.sf.jsqlparser.statement.select.FromItem; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.Join; -import net.sf.jsqlparser.statement.select.OrderByElement; -import net.sf.jsqlparser.statement.select.ParenthesedFromItem; -import net.sf.jsqlparser.statement.select.ParenthesedSelect; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; -import net.sf.jsqlparser.statement.select.SelectItem; -import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; -import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.*; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; @@ -623,6 +613,63 @@ public class SqlReplaceHelper { return selectStatement.toString(); } + public static void replaceSqlByPositions(Select select) { + if (select instanceof PlainSelect) { + PlainSelect plainSelect = (PlainSelect) select; + if (plainSelect.getSelectItems() != null) { + Map columnMap = new HashMap<>(); + for (int i = 0; i < plainSelect.getSelectItems().size(); i++) { + SelectItem selectItem = plainSelect.getSelectItems().get(i); + if (selectItem.getAlias() != null) { + columnMap.put(selectItem.getAlias().getName(), i + 1); + } else if (selectItem.getExpression() instanceof Column) { + Column column = (Column) selectItem.getExpression(); + columnMap.put(column.getColumnName(), i + 1); + } + } + if (plainSelect.getGroupBy() != null) { + ExpressionList groupByExpressionList = + plainSelect.getGroupBy().getGroupByExpressionList(); + List groupByExpressions = groupByExpressionList.getExpressions(); + for (Expression expression : groupByExpressions) { + if (expression instanceof Column) { + Column column = (Column) expression; + if (columnMap.containsKey(column.getColumnName())) { + column.setColumnName( + String.valueOf(columnMap.get(column.getColumnName()))); + } + } + } + } + if (plainSelect.getOrderByElements() != null) { + for (OrderByElement orderByElement : plainSelect.getOrderByElements()) { + if (orderByElement.getExpression() instanceof Column) { + Column column = (Column) orderByElement.getExpression(); + if (columnMap.containsKey(column.getColumnName())) { + orderByElement.setExpression( + new LongValue(columnMap.get(column.getColumnName()))); + } + } + } + } + } + } + if (select instanceof ParenthesedSelect) { + ParenthesedSelect parenthesedSelect = (ParenthesedSelect) select; + replaceSqlByPositions(parenthesedSelect.getSelect()); + } + if (select instanceof SetOperationList) { + SetOperationList setOperationList = (SetOperationList) select; + setOperationList.getSelects().forEach(s -> replaceSqlByPositions(s)); + } + } + + public static String replaceSqlByPositions(String sql) { + Select selectStatement = SqlSelectHelper.getSelect(sql); + replaceSqlByPositions(selectStatement); + return selectStatement.toString(); + } + private static void replacePlainSelectByExpr(PlainSelect plainSelect, Map replace) { QueryExpressionReplaceVisitor expressionReplaceVisitor = diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java index d2e8826f3..78cc8720e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java @@ -35,6 +35,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorWithHints; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.util.Litmus; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -461,4 +462,27 @@ public abstract class SemanticNode { } return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO); } + + public static List deduplicateNode(List listNode) { // List去重 + if (listNode == null) { + return null; + } + List uniqueElements = new ArrayList<>(); + for (SqlNode element : listNode) { + if (!containsElement(uniqueElements, element)) { + uniqueElements.add(element); + } + } + return uniqueElements; + } + + private static boolean containsElement(List list, SqlNode element) { // 检查List中是否含有某element + for (SqlNode i : list) { + if (i.equalsDeep(element, Litmus.IGNORE)) { + return true; + } + } + return false; + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java index 1c815e12c..bf3abfd03 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java @@ -66,6 +66,8 @@ public class FilterRender extends Renderer { tableView.getMeasure().add(SemanticNode.parse(metric, scope, engineType)); } } + tableView.setMeasure(SemanticNode.deduplicateNode(tableView.getMeasure())); + tableView.setDimension(SemanticNode.deduplicateNode(tableView.getDimension())); if (filterNode != null) { TableView filterView = new TableView(); filterView.setTable(SemanticNode.buildAs(Constants.DATASOURCE_TABLE_FILTER_PREFIX, diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index 6bc729efe..76cc6bf68 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -138,6 +138,8 @@ public class JoinRender extends Renderer { } } } + filterView.setMeasure(SemanticNode.deduplicateNode(filterView.getMeasure())); + filterView.setDimension(SemanticNode.deduplicateNode(filterView.getDimension())); super.tableView = filterView; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java index 29c990525..140d601d9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java @@ -94,8 +94,8 @@ public class SourceRender extends Renderer { datasource, schema, nonAgg, extendFields, dataSet, output, scope); } - output.setMeasure(deduplicateNode(output.getMeasure())); - dataSet.setMeasure(deduplicateNode(dataSet.getMeasure())); + output.setMeasure(SemanticNode.deduplicateNode(output.getMeasure())); + dataSet.setMeasure(SemanticNode.deduplicateNode(dataSet.getMeasure())); SqlNode tableNode = DataSourceNode.buildExtend(datasource, extendFields, scope); dataSet.setTable(tableNode); @@ -105,24 +105,7 @@ public class SourceRender extends Renderer { return output; } - private static List deduplicateNode(List listNode) { // List去重 - List uniqueElements = new ArrayList<>(); - for (SqlNode element : listNode) { - if (!containsElement(uniqueElements, element)) { - uniqueElements.add(element); - } - } - return uniqueElements; - } - private static boolean containsElement(List list, SqlNode element) { // 检查List中是否含有某element - for (SqlNode i : list) { - if (i.equalsDeep(element, Litmus.IGNORE)) { - return true; - } - } - return false; - } private static void buildDimension(String alias, String dimension, DataSource datasource, SemanticSchema schema, boolean nonAgg, Map extendFields, diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 55f3d45fd..3236356f5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -149,9 +149,14 @@ public class QueryReqConverter { } private AggOption getAggOption(QuerySqlReq databaseReq, List metricSchemas) { + String sql = databaseReq.getSql(); + if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) + && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { + log.debug("getAggOption simple sql set to DEFAULT"); + return AggOption.DEFAULT; + } // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" - String sql = databaseReq.getSql(); if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) || SqlSelectFunctionHelper.hasFunction(sql, "count") || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { @@ -171,11 +176,6 @@ public class QueryReqConverter { log.debug("getAggOption find null defaultAgg metric set to NATIVE"); return AggOption.OUTER; } - if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) - && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { - log.debug("getAggOption simple sql set to NATIVE"); - return AggOption.NATIVE; - } return AggOption.DEFAULT; } @@ -185,6 +185,8 @@ public class QueryReqConverter { String sql = querySqlReq.getSql(); log.debug("dataSetId:{},convert name to bizName before:{}", querySqlReq.getDataSetId(), sql); + sql = SqlReplaceHelper.replaceSqlByPositions(sql); + log.debug("replaceSqlByPositions:{}", sql); String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); log.debug("dataSetId:{},convert name to bizName after:{}", querySqlReq.getDataSetId(), replaceFields);