From 5474fe1d04dabebd4069acfe0f39e801002e5cd0 Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Thu, 20 Jun 2024 10:43:17 +0800 Subject: [PATCH] (improvement)(headless) fixed parser error (#1161) (#1166) --- .../core/parser/calcite/Configuration.java | 16 +++++++++- .../parser/calcite/schema/SchemaBuilder.java | 12 ++++---- .../calcite/sql/node/DataSourceNode.java | 30 +++++++------------ .../parser/calcite/sql/node/SemanticNode.java | 7 +++++ .../server/utils/QueryReqConverter.java | 2 +- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java index 1f39bab98..755efbbb2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.parser.calcite; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl; import com.tencent.supersonic.headless.core.parser.calcite.schema.ViewExpanderImpl; import com.tencent.supersonic.headless.core.utils.SqlDialectFactory; import java.util.ArrayList; @@ -27,6 +27,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.advise.SqlAdvisor; +import org.apache.calcite.sql.advise.SqlAdvisorValidator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.impl.SqlParserImpl; @@ -34,6 +36,7 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.apache.calcite.sql.validate.SqlValidatorWithHints; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; @@ -96,6 +99,13 @@ public class Configuration { Configuration.getValidatorConfig(engineType)); } + public static SqlValidatorWithHints getSqlValidatorWithHints(CalciteSchema rootSchema, EngineType engineTyp) { + return new SqlAdvisorValidator(SqlStdOperatorTable.instance(), + new CalciteCatalogReader(rootSchema, + Collections.singletonList(rootSchema.getName()), typeFactory, config), + typeFactory, SqlValidator.Config.DEFAULT); + } + public static SqlToRelConverter.Config getConverterConfig() { HintStrategyTable strategies = HintStrategyTable.builder().build(); return SqlToRelConverter.config() @@ -119,4 +129,8 @@ public class Configuration { getConverterConfig()); } + public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) { + return new SqlAdvisor(validator, getParserConfig(engineType)); + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index 5ff03cb71..4a09e4ed1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -80,12 +80,14 @@ public class SchemaBuilder { .withRowCount(1) .build(); if (Objects.nonNull(db) && !db.isEmpty()) { - SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db); - if (Objects.isNull(schemaPlus)) { - dataSetSchema.plus().add(db, new AbstractSchema()); - schemaPlus = dataSetSchema.plus().getSubSchema(db); + SchemaPlus dbPs = dataSetSchema.plus(); + for (String d : db.split("\\.")) { + if (dbPs.getSubSchema(d) == null) { + dbPs.add(d, new AbstractSchema()); + } + dbPs = dbPs.getSubSchema(d); } - schemaPlus.add(tb, srcTable); + dbPs.add(tb, srcTable); } else { dataSetSchema.add(tb, srcTable); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index 8977fb1cc..7dc4ccac7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; @@ -55,29 +56,20 @@ public class DataSourceNode extends SemanticNode { throw new Exception("DatasourceNode build error [tableSqlNode not found]"); } SqlNode source = getTable(sqlTable, scope, EngineType.fromString(datasource.getType())); - addSchema(scope, datasource, source); + addSchema(scope, datasource, sqlTable); return buildAs(datasource.getName(), source); } - private static void addSchema(SqlValidatorScope scope, DataSource datasource, SqlNode table) throws Exception { - Map parseInfo = SemanticNode.getDbTable(table); - if (!parseInfo.isEmpty() && parseInfo.containsKey(Constants.SQL_PARSER_TABLE)) { - Map> dbTbs = (Map>) parseInfo.get(Constants.SQL_PARSER_TABLE); - Map> fields = (Map>) parseInfo.get(Constants.SQL_PARSER_FIELD); - for (Map.Entry> entry : dbTbs.entrySet()) { - for (String dbTb : entry.getValue()) { - String[] dbTable = dbTb.split("\\."); - if (Objects.nonNull(dbTable) && dbTable.length > 0) { - String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0]; - String db = dbTable.length > 1 ? dbTable[0] : ""; - addSchemaTable(scope, datasource, db, tb, - fields.containsKey(entry.getKey()) ? fields.get(entry.getKey()) - : dbTbs.size() == 1 && fields.size() == 1 && fields.containsKey("") - ? fields.get("") - : new HashSet<>()); - } - } + private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) throws Exception { + Map> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table); + for (Map.Entry> entry : sqlTable.entrySet()) { + String tb = entry.getKey(); + String db = ""; + if (entry.getKey().indexOf(".") > 0) { + db = entry.getKey().substring(0, entry.getKey().indexOf(".")); + tb = entry.getKey().substring(entry.getKey().indexOf(".") + 1); } + addSchemaTable(scope, datasource, db, tb, entry.getValue()); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java index d9a8a5fe5..1258bb36a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java @@ -46,6 +46,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql.validate.SqlValidatorWithHints; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -75,6 +76,12 @@ public abstract class SemanticNode { } public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlValidatorWithHints sqlValidatorWithHints = Configuration.getSqlValidatorWithHints( + scope.getValidator().getCatalogReader().getRootSchema(), engineType); + if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords() + .contains(expression.toUpperCase())) { + expression = String.format("`%s`", expression); + } SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseExpression(); scope.validateExpr(sqlNode); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 2b2d712c2..860ce4450 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -156,7 +156,7 @@ public class QueryReqConverter { if (databaseReq.isInnerLayerNative()) { return AggOption.NATIVE; } - if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)) { + if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) { return AggOption.NATIVE; } long defaultAggNullCnt = metricSchemas.stream()