topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Fixed duplicate fields error (#1870)

Browse Source
This commit is contained in:
jipeli
2024-10-31 22:46:46 +08:00
committed by GitHub
parent 7773c82eea
commit 0cb1faaf53
6 changed files with 96 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java
View File

@@ -20,17 +20,7 @@ import net.sf.jsqlparser.expression.operators.relational.NotEqualsTo;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.FromItem;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.Join;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.ParenthesedFromItem;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
import net.sf.jsqlparser.statement.select.SetOperationList;
import net.sf.jsqlparser.statement.select.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
@@ -623,6 +613,63 @@ public class SqlReplaceHelper {
return selectStatement.toString();
}
public static void replaceSqlByPositions(Select select) {
if (select instanceof PlainSelect) {
PlainSelect plainSelect = (PlainSelect) select;
if (plainSelect.getSelectItems() != null) {
Map<String, Integer> columnMap = new HashMap<>();
for (int i = 0; i < plainSelect.getSelectItems().size(); i++) {
SelectItem selectItem = plainSelect.getSelectItems().get(i);
if (selectItem.getAlias() != null) {
columnMap.put(selectItem.getAlias().getName(), i + 1);
} else if (selectItem.getExpression() instanceof Column) {
Column column = (Column) selectItem.getExpression();
columnMap.put(column.getColumnName(), i + 1);
}
}
if (plainSelect.getGroupBy() != null) {
ExpressionList groupByExpressionList =
plainSelect.getGroupBy().getGroupByExpressionList();
List<Expression> groupByExpressions = groupByExpressionList.getExpressions();
for (Expression expression : groupByExpressions) {
if (expression instanceof Column) {
Column column = (Column) expression;
if (columnMap.containsKey(column.getColumnName())) {
column.setColumnName(
String.valueOf(columnMap.get(column.getColumnName())));
}
}
}
}
if (plainSelect.getOrderByElements() != null) {
for (OrderByElement orderByElement : plainSelect.getOrderByElements()) {
if (orderByElement.getExpression() instanceof Column) {
Column column = (Column) orderByElement.getExpression();
if (columnMap.containsKey(column.getColumnName())) {
orderByElement.setExpression(
new LongValue(columnMap.get(column.getColumnName())));
}
}
}
}
}
}
if (select instanceof ParenthesedSelect) {
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) select;
replaceSqlByPositions(parenthesedSelect.getSelect());
}
if (select instanceof SetOperationList) {
SetOperationList setOperationList = (SetOperationList) select;
setOperationList.getSelects().forEach(s -> replaceSqlByPositions(s));
}
}
public static String replaceSqlByPositions(String sql) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
replaceSqlByPositions(selectStatement);
return selectStatement.toString();
}
private static void replacePlainSelectByExpr(PlainSelect plainSelect,
Map<String, String> replace) {
QueryExpressionReplaceVisitor expressionReplaceVisitor =

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java
View File

@@ -35,6 +35,7 @@ import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorWithHints;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.util.Litmus;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -461,4 +462,27 @@ public abstract class SemanticNode {
}
return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO);
}
public static List<SqlNode> deduplicateNode(List<SqlNode> listNode) { // List<SqlNode>去重
if (listNode == null) {
return null;
}
List<SqlNode> uniqueElements = new ArrayList<>();
for (SqlNode element : listNode) {
if (!containsElement(uniqueElements, element)) {
uniqueElements.add(element);
}
}
return uniqueElements;
}
private static boolean containsElement(List<SqlNode> list, SqlNode element) { // 检查List<SqlNode>中是否含有某element
for (SqlNode i : list) {
if (i.equalsDeep(element, Litmus.IGNORE)) {
return true;
}
}
return false;
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java
View File

@@ -66,6 +66,8 @@ public class FilterRender extends Renderer {
tableView.getMeasure().add(SemanticNode.parse(metric, scope, engineType));
}
}
tableView.setMeasure(SemanticNode.deduplicateNode(tableView.getMeasure()));
tableView.setDimension(SemanticNode.deduplicateNode(tableView.getDimension()));
if (filterNode != null) {
TableView filterView = new TableView();
filterView.setTable(SemanticNode.buildAs(Constants.DATASOURCE_TABLE_FILTER_PREFIX,

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -138,6 +138,8 @@ public class JoinRender extends Renderer {
}
}
}
filterView.setMeasure(SemanticNode.deduplicateNode(filterView.getMeasure()));
filterView.setDimension(SemanticNode.deduplicateNode(filterView.getDimension()));
super.tableView = filterView;
}

21
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -94,8 +94,8 @@ public class SourceRender extends Renderer {
datasource, schema, nonAgg, extendFields, dataSet, output, scope);
}
output.setMeasure(deduplicateNode(output.getMeasure()));
dataSet.setMeasure(deduplicateNode(dataSet.getMeasure()));
output.setMeasure(SemanticNode.deduplicateNode(output.getMeasure()));
dataSet.setMeasure(SemanticNode.deduplicateNode(dataSet.getMeasure()));
SqlNode tableNode = DataSourceNode.buildExtend(datasource, extendFields, scope);
dataSet.setTable(tableNode);
@@ -105,24 +105,7 @@ public class SourceRender extends Renderer {
return output;
}
private static List<SqlNode> deduplicateNode(List<SqlNode> listNode) { // List<SqlNode>去重
List<SqlNode> uniqueElements = new ArrayList<>();
for (SqlNode element : listNode) {
if (!containsElement(uniqueElements, element)) {
uniqueElements.add(element);
}
}
return uniqueElements;
}
private static boolean containsElement(List<SqlNode> list, SqlNode element) { // 检查List<SqlNode>中是否含有某element
for (SqlNode i : list) {
if (i.equalsDeep(element, Litmus.IGNORE)) {
return true;
}
}
return false;
}
private static void buildDimension(String alias, String dimension, DataSource datasource,
SemanticSchema schema, boolean nonAgg, Map<String, String> extendFields,

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -149,9 +149,14 @@ public class QueryReqConverter {
}
private AggOption getAggOption(QuerySqlReq databaseReq, List<MetricSchemaResp> metricSchemas) {
String sql = databaseReq.getSql();
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to DEFAULT");
return AggOption.DEFAULT;
}
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
@@ -171,11 +176,6 @@ public class QueryReqConverter {
log.debug("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.OUTER;
}
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to NATIVE");
return AggOption.NATIVE;
}
return AggOption.DEFAULT;
}
@@ -185,6 +185,8 @@ public class QueryReqConverter {
String sql = querySqlReq.getSql();
log.debug("dataSetId:{},convert name to bizName before:{}", querySqlReq.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceSqlByPositions(sql);
log.debug("replaceSqlByPositions:{}", sql);
String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.debug("dataSetId:{},convert name to bizName after:{}", querySqlReq.getDataSetId(),
replaceFields);