topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) fixed parser error (#1161) (#1166)

Browse Source
This commit is contained in:
jipeli
2024-06-20 10:43:17 +08:00
committed by GitHub
parent 633c7eae07
commit 5474fe1d04
5 changed files with 41 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.parser.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl;
import com.tencent.supersonic.headless.core.parser.calcite.schema.ViewExpanderImpl;
import com.tencent.supersonic.headless.core.utils.SqlDialectFactory;
import java.util.ArrayList;
@@ -27,6 +27,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.advise.SqlAdvisor;
import org.apache.calcite.sql.advise.SqlAdvisorValidator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.impl.SqlParserImpl;
@@ -34,6 +36,7 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.sql.validate.SqlValidatorWithHints;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
@@ -96,6 +99,13 @@ public class Configuration {
Configuration.getValidatorConfig(engineType));
}
public static SqlValidatorWithHints getSqlValidatorWithHints(CalciteSchema rootSchema, EngineType engineTyp) {
return new SqlAdvisorValidator(SqlStdOperatorTable.instance(),
new CalciteCatalogReader(rootSchema,
Collections.singletonList(rootSchema.getName()), typeFactory, config),
typeFactory, SqlValidator.Config.DEFAULT);
}
public static SqlToRelConverter.Config getConverterConfig() {
HintStrategyTable strategies = HintStrategyTable.builder().build();
return SqlToRelConverter.config()
@@ -119,4 +129,8 @@ public class Configuration {
getConverterConfig());
}
public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) {
return new SqlAdvisor(validator, getParserConfig(engineType));
}
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java
View File

@@ -80,12 +80,14 @@ public class SchemaBuilder {
.withRowCount(1)
.build();
if (Objects.nonNull(db) && !db.isEmpty()) {
SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db);
if (Objects.isNull(schemaPlus)) {
dataSetSchema.plus().add(db, new AbstractSchema());
schemaPlus = dataSetSchema.plus().getSubSchema(db);
SchemaPlus dbPs = dataSetSchema.plus();
for (String d : db.split("\\.")) {
if (dbPs.getSubSchema(d) == null) {
dbPs.add(d, new AbstractSchema());
}
dbPs = dbPs.getSubSchema(d);
}
schemaPlus.add(tb, srcTable);
dbPs.add(tb, srcTable);
} else {
dataSetSchema.add(tb, srcTable);
}

30
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
@@ -55,29 +56,20 @@ public class DataSourceNode extends SemanticNode {
throw new Exception("DatasourceNode build error [tableSqlNode not found]");
}
SqlNode source = getTable(sqlTable, scope, EngineType.fromString(datasource.getType()));
addSchema(scope, datasource, source);
addSchema(scope, datasource, sqlTable);
return buildAs(datasource.getName(), source);
}
private static void addSchema(SqlValidatorScope scope, DataSource datasource, SqlNode table) throws Exception {
Map<String, Object> parseInfo = SemanticNode.getDbTable(table);
if (!parseInfo.isEmpty() && parseInfo.containsKey(Constants.SQL_PARSER_TABLE)) {
Map<String, Set<String>> dbTbs = (Map<String, Set<String>>) parseInfo.get(Constants.SQL_PARSER_TABLE);
Map<String, Set<String>> fields = (Map<String, Set<String>>) parseInfo.get(Constants.SQL_PARSER_FIELD);
for (Map.Entry<String, Set<String>> entry : dbTbs.entrySet()) {
for (String dbTb : entry.getValue()) {
String[] dbTable = dbTb.split("\\.");
if (Objects.nonNull(dbTable) && dbTable.length > 0) {
String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0];
String db = dbTable.length > 1 ? dbTable[0] : "";
addSchemaTable(scope, datasource, db, tb,
fields.containsKey(entry.getKey()) ? fields.get(entry.getKey())
: dbTbs.size() == 1 && fields.size() == 1 && fields.containsKey("")
? fields.get("")
: new HashSet<>());
}
}
private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) throws Exception {
Map<String, Set<String>> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table);
for (Map.Entry<String, Set<String>> entry : sqlTable.entrySet()) {
String tb = entry.getKey();
String db = "";
if (entry.getKey().indexOf(".") > 0) {
db = entry.getKey().substring(0, entry.getKey().indexOf("."));
tb = entry.getKey().substring(entry.getKey().indexOf(".") + 1);
}
addSchemaTable(scope, datasource, db, tb, entry.getValue());
}
}

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java
View File

@@ -46,6 +46,7 @@ import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorWithHints;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -75,6 +76,12 @@ public abstract class SemanticNode {
}
public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) throws Exception {
SqlValidatorWithHints sqlValidatorWithHints = Configuration.getSqlValidatorWithHints(
scope.getValidator().getCatalogReader().getRootSchema(), engineType);
if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords()
.contains(expression.toUpperCase())) {
expression = String.format("`%s`", expression);
}
SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType));
SqlNode sqlNode = sqlParser.parseExpression();
scope.validateExpr(sqlNode);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -156,7 +156,7 @@ public class QueryReqConverter {
if (databaseReq.isInnerLayerNative()) {
return AggOption.NATIVE;
}
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)) {
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.NATIVE;
}
long defaultAggNullCnt = metricSchemas.stream()