topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(semantic) add materialization optimizer (#239)

Browse Source 
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-10-16 22:07:45 +08:00
committed by GitHub
parent 5b8fde70ca
commit 40ba179703
126 changed files with 9172 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java
View File

@@ -11,6 +11,7 @@ public enum AggOperatorEnum {
SUM("SUM"),
COUNT_DISTINCT("COUNT_DISTINCT"),
DISTINCT("DISTINCT"),
TOPN("TOPN"),

45
common/src/main/java/com/tencent/supersonic/common/pojo/enums/DataTypeEnums.java Normal file
View File

@@ -0,0 +1,45 @@
package com.tencent.supersonic.common.pojo.enums;
public enum DataTypeEnums {
ARRAY("ARRAY"),
MAP("MAP"),
JSON("JSON"),
VARCHAR("VARCHAR"),
DATE("DATE"),
BIGINT("BIGINT"),
INT("INT"),
DOUBLE("DOUBLE"),
FLOAT("FLOAT"),
DECIMAL("DECIMAL"),
UNKNOWN("unknown");
private String type;
DataTypeEnums(String type) {
this.type = type;
}
public String getType() {
return type;
}
public static DataTypeEnums of(String type) {
for (DataTypeEnums typeEnum : DataTypeEnums.values()) {
if (typeEnum.getType().equalsIgnoreCase(type)) {
return typeEnum;
}
}
return DataTypeEnums.UNKNOWN;
}
}

8
common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.common.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@@ -17,4 +18,11 @@ public class StringUtil {
return String.format(SPACE_WRAPPER, value);
}
public static String formatSqlQuota(String where) {
if (StringUtils.isEmpty(where)) {
return where;
}
return where.replace("\"", "\\\\\"");
}
}

69
common/src/main/java/com/tencent/supersonic/common/util/calcite/SqlParseUtils.java
View File

@@ -1,7 +1,10 @@
package com.tencent.supersonic.common.util.calcite;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -13,6 +16,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.fun.SqlCase;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
@@ -40,7 +44,6 @@ public class SqlParseUtils {
handlerSQL(sqlNode, sqlParserInfo);
sqlParserInfo.setAllFields(sqlParserInfo.getAllFields().stream().distinct().collect(Collectors.toList()));
sqlParserInfo.setSelectFields(
@@ -298,5 +301,69 @@ public class SqlParseUtils {
return null;
}
public static Set<String> getFilterField(String where) {
Set<String> result = new HashSet<>();
try {
SqlParser parser = SqlParser.create(where);
SqlNode sqlNode = parser.parseExpression();
getFieldByExpression(sqlNode, result);
return result;
} catch (SqlParseException e) {
throw new RuntimeException("getSqlParseInfo", e);
}
}
public static void getFieldByExpression(SqlNode sqlNode, Set<String> fields) {
if (sqlNode instanceof SqlIdentifier) {
SqlIdentifier sqlIdentifier = (SqlIdentifier) sqlNode;
fields.add(sqlIdentifier.names.get(0).toLowerCase());
return;
}
if (sqlNode instanceof SqlBasicCall) {
SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode;
for (SqlNode operand : sqlBasicCall.getOperandList()) {
getFieldByExpression(operand, fields);
}
}
}
public static Map getCaseExprFields(String expr) {
SqlParser parser = SqlParser.create(expr);
Map<String, String> ret = new HashMap();
try {
SqlNode sqlNodeCase = parser.parseExpression();
if (sqlNodeCase instanceof SqlCase) {
SqlCase sqlCase = (SqlCase) sqlNodeCase;
if (CollectionUtils.isEmpty(sqlCase.getThenOperands()) || CollectionUtils.isEmpty(
sqlCase.getWhenOperands())) {
return ret;
}
SqlDialect dialect = new S2MysqlSqlDialect(S2MysqlSqlDialect.DEFAULT_CONTEXT);
int i = 0;
for (SqlNode sqlNode : sqlCase.getWhenOperands().getList()) {
if (sqlNode instanceof SqlBasicCall) {
SqlBasicCall when = (SqlBasicCall) sqlNode;
if (!org.springframework.util.CollectionUtils.isEmpty(when.getOperandList())
&& when.getOperandList().size() > 1) {
String value = when.getOperandList().get(1).toSqlString(dialect).getSql();
if (sqlCase.getThenOperands().get(i) != null) {
if (sqlCase.getThenOperands().get(i) instanceof SqlIdentifier) {
SqlIdentifier sqlIdentifier = (SqlIdentifier) sqlCase.getThenOperands().get(i);
String field = sqlIdentifier.getSimple();
ret.put(value, field);
}
}
}
}
i++;
}
}
} catch (SqlParseException e) {
throw new RuntimeException("getSqlParseInfo", e);
}
return ret;
}
}

96
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FilterRemoveVisitor.java Normal file
View File

@@ -0,0 +1,96 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.lang3.StringUtils;
public class FilterRemoveVisitor extends ExpressionVisitorAdapter {
private List<String> filedNames;
public FilterRemoveVisitor(List<String> filedNames) {
this.filedNames = filedNames;
}
private boolean isRemove(Expression leftExpression) {
if (!(leftExpression instanceof Column)) {
return false;
}
Column leftColumnName = (Column) leftExpression;
String columnName = leftColumnName.getColumnName();
if (StringUtils.isEmpty(columnName)) {
return false;
}
if (!filedNames.contains(columnName)) {
return false;
}
return true;
}
@Override
public void visit(EqualsTo expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(MinorThan expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(0L));
}
@Override
public void visit(MinorThanEquals expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(GreaterThan expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(0L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(GreaterThanEquals expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
@Override
public void visit(InExpression expr) {
if (!isRemove(expr.getLeftExpression())) {
return;
}
expr.setNot(false);
expr.setRightExpression(new LongValue(1L));
expr.setLeftExpression(new LongValue(1L));
}
}

19
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -96,5 +97,23 @@ public class SqlParserRemoveHelper {
}
}
public static String removeWhere(String sql, List<String> fields) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
Expression where = plainSelect.getWhere();
if (where == null) {
return sql;
} else {
where.accept(new FilterRemoveVisitor(fields));
plainSelect.setWhere(where);
}
return selectStatement.toString();
}
}

62
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java
View File

@@ -10,10 +10,14 @@ import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
import net.sf.jsqlparser.expression.operators.conditional.XorExpression;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
@@ -27,6 +31,7 @@ import net.sf.jsqlparser.statement.select.SelectExpressionItem;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
import net.sf.jsqlparser.statement.select.SubSelect;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.springframework.util.CollectionUtils;
/**
@@ -259,14 +264,7 @@ public class SqlParserSelectHelper {
}
public static String getTableName(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
Table table = (Table) plainSelect.getFromItem();
Table table = getTable(sql);
return table.getName();
}
@@ -341,5 +339,53 @@ public class SqlParserSelectHelper {
}
return "";
}
public static Expression getTimeFilter(List<ImmutablePair<String, String>> times, String columnBegin,
String columnEnd) {
Expression expression = null;
for (ImmutablePair<String, String> t : times) {
Expression expr = null;
ComparisonOperator left = new MinorThanEquals();
if (t.left.equals(t.right)) {
left.setLeftExpression(new Column(columnBegin));
left.setRightExpression(new StringValue(t.left));
ComparisonOperator right = new GreaterThan();
right.setLeftExpression(new Column(columnEnd));
right.setRightExpression(new StringValue(t.right));
expr = new AndExpression(left, right);
} else {
left.setLeftExpression(new StringValue(t.left));
left.setRightExpression(new Column(columnEnd));
ComparisonOperator right = new GreaterThanEquals();
right.setLeftExpression(new StringValue(t.right));
right.setRightExpression(new Column(columnBegin));
expr = new AndExpression(left, right);
}
if (expression == null) {
expression = expr;
continue;
}
expression = new OrExpression(expression, expr);
}
return expression;
}
public static Table getTable(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
Table table = (Table) plainSelect.getFromItem();
return table;
}
public static String getDbTableName(String sql) {
Table table = getTable(sql);
return table.getFullyQualifiedName();
}
}