From 9c6bd7cf19597dcdf64507d3d661e6e104094626 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:32:28 +0800 Subject: [PATCH] (improvement)(Headless) Support optimizing and generating SQL based on the engine type. (#666) --- .../semantic/LocalSemanticInterpreter.java | 2 +- .../headless/api/enums/EngineType.java | 10 +++ .../core/parser/calcite/CalciteSqlParser.java | 8 ++- .../core/parser/calcite/Configuration.java | 35 ++++++----- .../parser/calcite/planner/AggPlanner.java | 32 +++++----- .../core/parser/calcite/planner/Planner.java | 5 +- .../parser/calcite/schema/SchemaBuilder.java | 4 +- ...mance.java => SemanticSqlConformance.java} | 2 +- .../calcite/schema/SemanticSqlDialect.java | 11 +--- ...l.java => SemanticSqlTypeFactoryImpl.java} | 4 +- .../core/parser/calcite/sql/Renderer.java | 28 +++++---- .../calcite/sql/node/AggFunctionNode.java | 11 ++-- .../calcite/sql/node/DataSourceNode.java | 31 ++++++---- .../calcite/sql/node/DimensionNode.java | 50 +++++++-------- .../parser/calcite/sql/node/IdentifyNode.java | 5 +- .../parser/calcite/sql/node/MeasureNode.java | 43 ++++--------- .../parser/calcite/sql/node/MetricNode.java | 7 ++- .../parser/calcite/sql/node/SemanticNode.java | 34 +++++----- .../calcite/sql/render/FilterRender.java | 11 ++-- .../parser/calcite/sql/render/JoinRender.java | 60 ++++++++++-------- .../calcite/sql/render/OutputRender.java | 14 +++-- .../calcite/sql/render/SourceRender.java | 46 +++++++------- .../converter/CalculateAggConverter.java | 2 +- .../core/utils/SqlDialectFactory.java | 48 ++++++++++++++ .../pojo/PostgresqlParametersBuilder.java | 12 +--- .../headless/server/rest/QueryController.java | 1 - .../headless/server/service/QueryService.java | 2 +- .../server/service/impl/QueryServiceImpl.java | 62 +++++-------------- .../server/utils/QueryReqConverter.java | 2 +- .../calcite/HeadlessParserServiceTest.java | 4 +- 30 files changed, 309 insertions(+), 277 deletions(-) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/{HeadlessSqlConformance.java => SemanticSqlConformance.java} (98%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/{HeadlessSqlTypeFactoryImpl.java => SemanticSqlTypeFactoryImpl.java} (76%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java index 920f7e35d..f497fc633 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java @@ -67,7 +67,7 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @SneakyThrows public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { queryService = ContextUtils.getBean(QueryService.class); - Object object = queryService.queryBySql(querySQLReq, user); + SemanticQueryResp object = queryService.queryBySql(querySQLReq, user); return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java index 293d5d397..7e3b2a23f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java @@ -28,4 +28,14 @@ public enum EngineType { public String getName() { return name; } + + public static EngineType fromString(String value) { + for (EngineType engineType : EngineType.values()) { + if (engineType.name().equalsIgnoreCase(value)) { + return engineType; + } + } + throw new IllegalArgumentException("Invalid value: " + value); + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java index a45dab304..77534b5c6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; @@ -31,13 +32,15 @@ public class CalciteSqlParser implements SqlParser { SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); AggPlanner aggBuilder = new AggPlanner(semanticSchema); aggBuilder.explain(queryStatement, isAgg); - queryStatement.setSql(aggBuilder.getSql()); + EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); + queryStatement.setSql(aggBuilder.getSql(engineType)); queryStatement.setSourceId(aggBuilder.getSourceId()); if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize() && Objects.nonNull(queryStatement.getViewAlias()) && !queryStatement.getViewAlias().isEmpty()) { // simplify model sql with query sql String simplifySql = aggBuilder.simplify( - getSqlByView(aggBuilder.getSql(), queryStatement.getViewSql(), queryStatement.getViewAlias())); + getSqlByView(aggBuilder.getSql(engineType), queryStatement.getViewSql(), + queryStatement.getViewAlias()), engineType); if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { log.info("simplifySql [{}]", simplifySql); queryStatement.setViewSimplifySql(simplifySql); @@ -48,6 +51,7 @@ public class CalciteSqlParser implements SqlParser { private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) { SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build(); + semanticSchema.setSemanticModel(semanticModel); semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDimension(semanticModel.getDimensionMap()); semanticSchema.setMetric(semanticModel.getMetrics()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java index 7e918265b..05868881b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/Configuration.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.headless.core.parser.calcite; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSqlTypeFactoryImpl; +import com.tencent.supersonic.headless.api.enums.EngineType; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlTypeFactoryImpl; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.schema.ViewExpanderImpl; +import com.tencent.supersonic.headless.core.utils.SqlDialectFactory; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,13 +44,17 @@ import org.apache.calcite.tools.Frameworks; public class Configuration { public static Properties configProperties = new Properties(); - public static RelDataTypeFactory typeFactory = new HeadlessSqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + public static RelDataTypeFactory typeFactory = new SemanticSqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties); - public static SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT - .withConformance(SemanticSqlDialect.DEFAULT.getConformance()) - .withDefaultNullCollation(config.defaultNullCollation()) - .withLenientOperatorLookup(true); + + public static SqlValidator.Config getValidatorConfig(EngineType engineType) { + SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); + return SqlValidator.Config.DEFAULT + .withConformance(sqlDialect.getConformance()) + .withDefaultNullCollation(config.defaultNullCollation()) + .withLenientOperatorLookup(true); + } static { configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString()); @@ -57,8 +63,10 @@ public class Configuration { } - public static SqlParser.Config getParserConfig() { + public static SqlParser.Config getParserConfig(EngineType engineType) { CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties); + SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); + SqlParser.ConfigBuilder parserConfig = SqlParser.configBuilder(); parserConfig.setCaseSensitive(config.caseSensitive()); parserConfig.setUnquotedCasing(config.unquotedCasing()); @@ -71,12 +79,12 @@ public class Configuration { .setQuoting(Quoting.SINGLE_QUOTE) .setQuotedCasing(Casing.TO_UPPER) .setUnquotedCasing(Casing.TO_UPPER) - .setConformance(SemanticSqlDialect.DEFAULT.getConformance()) + .setConformance(sqlDialect.getConformance()) .setLex(Lex.BIG_QUERY); return parserConfig.build(); } - public static SqlValidator getSqlValidator(CalciteSchema rootSchema) { + public static SqlValidator getSqlValidator(CalciteSchema rootSchema, EngineType engineType) { List tables = new ArrayList<>(); tables.add(SqlStdOperatorTable.instance()); SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); @@ -87,9 +95,8 @@ public class Configuration { typeFactory, config ); - SqlValidator validator = SqlValidatorUtil.newValidator(operatorTable, catalogReader, typeFactory, - validatorConfig); - return validator; + return SqlValidatorUtil.newValidator(operatorTable, catalogReader, typeFactory, + Configuration.getValidatorConfig(engineType)); } public static SqlToRelConverter.Config getConverterConfig() { @@ -102,11 +109,11 @@ public class Configuration { } public static SqlToRelConverter getSqlToRelConverter(SqlValidatorScope scope, SqlValidator sqlValidator, - RelOptPlanner relOptPlanner) { + RelOptPlanner relOptPlanner, EngineType engineType) { RexBuilder rexBuilder = new RexBuilder(typeFactory); RelOptCluster cluster = RelOptCluster.create(relOptPlanner, rexBuilder); FrameworkConfig fromworkConfig = Frameworks.newConfigBuilder() - .parserConfig(getParserConfig()) + .parserConfig(getParserConfig(engineType)) .defaultSchema(scope.getValidator().getCatalogReader().getRootSchema().plus()) .build(); return new SqlToRelConverter(new ViewExpanderImpl(), diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java index 69a1be47b..98fb3f1a1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java @@ -2,12 +2,13 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode; @@ -15,6 +16,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender; +import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import java.util.ArrayList; import java.util.LinkedList; @@ -113,12 +115,14 @@ public class AggPlanner implements Planner { // build a parse Node parse(); // optimizer - optimize(); + Database database = queryStatement.getSemanticModel().getDatabase(); + EngineType engineType = EngineType.fromString(database.getType()); + optimize(engineType); } @Override - public String getSql() { - return SemanticNode.getSql(parserNode); + public String getSql(EngineType engineType) { + return SemanticNode.getSql(parserNode, engineType); } @Override @@ -127,26 +131,26 @@ public class AggPlanner implements Planner { } @Override - public String simplify(String sql) { - return optimize(sql); + public String simplify(String sql, EngineType engineType) { + return optimize(sql, engineType); } - public void optimize() { + public void optimize(EngineType engineType) { if (Objects.isNull(schema.getRuntimeOptions()) || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) || !schema.getRuntimeOptions().getEnableOptimize()) { return; } - SqlNode optimizeNode = optimizeSql(SemanticNode.getSql(parserNode)); + SqlNode optimizeNode = optimizeSql(SemanticNode.getSql(parserNode, engineType), engineType); if (Objects.nonNull(optimizeNode)) { parserNode = optimizeNode; } } - public String optimize(String sql) { + public String optimize(String sql, EngineType engineType) { try { - SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig()).parseStmt(); + SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); if (Objects.nonNull(sqlNode)) { - return SemanticNode.getSql(SemanticNode.optimize(scope, schema, sqlNode)); + return SemanticNode.getSql(SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType); } } catch (Exception e) { log.error("optimize error {}", e); @@ -154,12 +158,12 @@ public class AggPlanner implements Planner { return ""; } - private SqlNode optimizeSql(String sql) { + private SqlNode optimizeSql(String sql, EngineType engineType) { try { log.info("before optimize:[{}]", sql); - SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig()).parseStmt(); + SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); if (Objects.nonNull(sqlNode)) { - return SemanticNode.optimize(scope, schema, sqlNode); + return SemanticNode.optimize(scope, schema, sqlNode, engineType); } } catch (Exception e) { log.error("optimize error {}", e); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java index 035f5e37a..164e7d51e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** @@ -11,9 +12,9 @@ public interface Planner { public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception; - public String getSql(); + public String getSql(EngineType enginType); public String getSourceId(); - public String simplify(String sql); + public String simplify(String sql, EngineType engineType); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index d4f11cef6..b7146515c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.schema; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.sql.S2SQLSqlValidatorImpl; @@ -37,8 +38,9 @@ public class SchemaBuilder { Configuration.typeFactory, Configuration.config ); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader, - Configuration.typeFactory, Configuration.validatorConfig); + Configuration.typeFactory, Configuration.getValidatorConfig(engineType)); return new ParameterScope(s2SQLSqlValidator, nameToTypeMap); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlConformance.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlConformance.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlConformance.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlConformance.java index 9c96eb01b..91f4cbcd8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlConformance.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlConformance.java @@ -7,7 +7,7 @@ import org.apache.calcite.sql.validate.SqlConformanceEnum; /** * customize the SqlConformance */ -public class HeadlessSqlConformance implements SqlConformance { +public class SemanticSqlConformance implements SqlConformance { @Override public boolean isLiberal() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlDialect.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlDialect.java index 1578a855d..27269090f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlDialect.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlDialect.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.schema; import com.google.common.base.Preconditions; -import org.apache.calcite.avatica.util.Casing; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlIntervalLiteral; import org.apache.calcite.sql.SqlNode; @@ -14,15 +13,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; */ public class SemanticSqlDialect extends SqlDialect { - public static final Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT - .withDatabaseProduct(DatabaseProduct.BIG_QUERY) - .withLiteralQuoteString("'") - .withLiteralEscapedQuoteString("''") - .withUnquotedCasing(Casing.UNCHANGED) - .withQuotedCasing(Casing.UNCHANGED) - .withCaseSensitive(false); - public static final SqlDialect DEFAULT = new SemanticSqlDialect(DEFAULT_CONTEXT); - private static final SqlConformance tagTdwSqlConformance = new HeadlessSqlConformance(); + private static final SqlConformance tagTdwSqlConformance = new SemanticSqlConformance(); public SemanticSqlDialect(Context context) { super(context); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlTypeFactoryImpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlTypeFactoryImpl.java similarity index 76% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlTypeFactoryImpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlTypeFactoryImpl.java index 450940b9a..d86073564 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSqlTypeFactoryImpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlTypeFactoryImpl.java @@ -7,9 +7,9 @@ import org.apache.calcite.sql.type.SqlTypeFactoryImpl; /** * customize the SqlTypeFactoryImpl */ -public class HeadlessSqlTypeFactoryImpl extends SqlTypeFactoryImpl { +public class SemanticSqlTypeFactoryImpl extends SqlTypeFactoryImpl { - public HeadlessSqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { + public SemanticSqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { super(typeSystem); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java index 29bb62b9f..e70557b5b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MeasureNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MetricNode; @@ -47,27 +48,31 @@ public abstract class Renderer { } public static MetricNode buildMetricNode(String metric, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg, String alias) throws Exception { + SemanticSchema schema, boolean nonAgg, String alias) throws Exception { Optional metricOpt = getMetricByName(metric, schema); MetricNode metricNode = new MetricNode(); + EngineType engineType = EngineType.fromString(datasource.getType()); if (metricOpt.isPresent()) { metricNode.setMetric(metricOpt.get()); for (Measure m : metricOpt.get().getMetricTypeParams().getMeasures()) { Optional measure = getMeasureByName(m.getName(), datasource); if (measure.isPresent()) { metricNode.getNonAggNode() - .put(measure.get().getName(), MeasureNode.buildNonAgg(alias, measure.get(), scope)); + .put(measure.get().getName(), + MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType)); metricNode.getAggNode() - .put(measure.get().getName(), MeasureNode.buildAgg(measure.get(), nonAgg, scope)); + .put(measure.get().getName(), + MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType)); metricNode.getAggFunction().put(measure.get().getName(), measure.get().getAgg()); } else { - metricNode.getNonAggNode().put(m.getName(), MeasureNode.buildNonAgg(alias, m, scope)); - metricNode.getAggNode().put(m.getName(), MeasureNode.buildAgg(m, nonAgg, scope)); + metricNode.getNonAggNode().put(m.getName(), MeasureNode.buildNonAgg(alias, m, scope, engineType)); + metricNode.getAggNode().put(m.getName(), MeasureNode.buildAgg(m, nonAgg, scope, engineType)); metricNode.getAggFunction().put(m.getName(), m.getAgg()); } if (m.getConstraint() != null && !m.getConstraint().isEmpty()) { - metricNode.getMeasureFilter().put(m.getName(), SemanticNode.parse(m.getConstraint(), scope)); + metricNode.getMeasureFilter() + .put(m.getName(), SemanticNode.parse(m.getConstraint(), scope, engineType)); } } return metricNode; @@ -75,13 +80,14 @@ public abstract class Renderer { Optional measure = getMeasureByName(metric, datasource); if (measure.isPresent()) { metricNode.getNonAggNode() - .put(measure.get().getName(), MeasureNode.buildNonAgg(alias, measure.get(), scope)); - metricNode.getAggNode().put(measure.get().getName(), MeasureNode.buildAgg(measure.get(), nonAgg, scope)); + .put(measure.get().getName(), MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType)); + metricNode.getAggNode() + .put(measure.get().getName(), MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType)); metricNode.getAggFunction().put(measure.get().getName(), measure.get().getAgg()); if (measure.get().getConstraint() != null && !measure.get().getConstraint().isEmpty()) { - metricNode.getMeasureFilter() - .put(measure.get().getName(), SemanticNode.parse(measure.get().getConstraint(), scope)); + metricNode.getMeasureFilter().put(measure.get().getName(), + SemanticNode.parse(measure.get().getConstraint(), scope, engineType)); } } return metricNode; @@ -105,5 +111,5 @@ public abstract class Renderer { } public abstract void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception; + SemanticSchema schema, boolean nonAgg) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java index d311150cf..d385291b3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java @@ -1,19 +1,22 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import java.util.Objects; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; public class AggFunctionNode extends SemanticNode { - public static SqlNode build(String agg, String name, SqlValidatorScope scope) throws Exception { + public static SqlNode build(String agg, String name, SqlValidatorScope scope, EngineType engineType) + throws Exception { if (Objects.isNull(agg) || agg.isEmpty()) { - return parse(name, scope); + return parse(name, scope, engineType); } if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) { - return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope); + return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope, + engineType); } - return parse(agg + " ( " + name + " ) ", scope); + return parse(agg + " ( " + name + " ) ", scope, engineType); } public static enum AggFunction { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index 522f509f7..69e22b1b1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -54,7 +54,7 @@ public class DataSourceNode extends SemanticNode { if (sqlTable.isEmpty()) { throw new Exception("DatasourceNode build error [tableSqlNode not found]"); } - SqlNode source = getTable(sqlTable, scope); + SqlNode source = getTable(sqlTable, scope, EngineType.fromString(datasource.getType())); addSchema(scope, datasource, source); return buildAs(datasource.getName(), source); } @@ -87,8 +87,9 @@ public class DataSourceNode extends SemanticNode { Set dateInfo = new HashSet<>(); Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); + EngineType engineType = EngineType.fromString(datasource.getType()); for (Dimension d : datasource.getDimensions()) { - List identifiers = expand(SemanticNode.parse(d.getExpr(), scope), scope); + List identifiers = expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope); identifiers.stream().forEach(i -> dimensions.add(i.toString())); dimensions.add(d.getName()); } @@ -96,7 +97,7 @@ public class DataSourceNode extends SemanticNode { dimensions.add(i.getName()); } for (Measure m : datasource.getMeasures()) { - List identifiers = expand(SemanticNode.parse(m.getExpr(), scope), scope); + List identifiers = expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope); identifiers.stream().forEach(i -> { if (!dimensions.contains(i.toString())) { metrics.add(i.toString()); @@ -124,15 +125,17 @@ public class DataSourceNode extends SemanticNode { if (CollectionUtils.isEmpty(exprList)) { return build(datasource, scope); } + EngineType engineType = EngineType.fromString(datasource.getType()); SqlNode view = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope), - new SqlNodeList(getExtendField(exprList, scope), SqlParserPos.ZERO)), SqlParserPos.ZERO); + new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO); return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, view); } - public static List getExtendField(Set exprList, SqlValidatorScope scope) throws Exception { + public static List getExtendField(Set exprList, SqlValidatorScope scope, EngineType engineType) + throws Exception { List sqlNodeList = new ArrayList<>(); for (String expr : exprList) { - sqlNodeList.add(parse(expr, scope)); + sqlNodeList.add(parse(expr, scope, engineType)); sqlNodeList.add(new SqlDataTypeSpec( new SqlUserDefinedTypeNameSpec(expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, SqlParserPos.ZERO), SqlParserPos.ZERO)); @@ -140,8 +143,8 @@ public class DataSourceNode extends SemanticNode { return sqlNodeList; } - private static SqlNode getTable(String sqlQuery, SqlValidatorScope scope) throws Exception { - SqlParser sqlParser = SqlParser.create(sqlQuery, Configuration.getParserConfig()); + private static SqlNode getTable(String sqlQuery, SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlParser sqlParser = SqlParser.create(sqlQuery, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseQuery(); scope.validateExpr(sqlNode); return sqlNode; @@ -166,9 +169,10 @@ public class DataSourceNode extends SemanticNode { public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand, Set queryDimension, List measures, SqlValidatorScope scope) throws Exception { + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { Set filterConditions = new HashSet<>(); - FilterNode.getFilterField(parse(metricCommand.getWhere(), scope), filterConditions); + FilterNode.getFilterField(parse(metricCommand.getWhere(), scope, engineType), filterConditions); Set queryMeasures = new HashSet<>(measures); Set schemaMetricName = schema.getMetrics().stream() .map(m -> m.getName()).collect(Collectors.toSet()); @@ -223,8 +227,10 @@ public class DataSourceNode extends SemanticNode { } filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(dimension); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, scope); - boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope); + boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope, + engineType); if (isAllMatch) { log.info("baseDataSource match all "); return dataSources; @@ -260,7 +266,8 @@ public class DataSourceNode extends SemanticNode { List measures, Set dimension, MetricQueryReq metricCommand, - SqlValidatorScope scope) throws Exception { + SqlValidatorScope scope, + EngineType engineType) throws Exception { boolean isAllMatch = true; sourceMeasure.retainAll(measures); if (sourceMeasure.size() < measures.size()) { @@ -285,7 +292,7 @@ public class DataSourceNode extends SemanticNode { if (metricCommand.getWhere() != null && !metricCommand.getWhere().isEmpty()) { Set whereFields = new HashSet<>(); - SqlNode sqlNode = parse(metricCommand.getWhere(), scope); + SqlNode sqlNode = parse(metricCommand.getWhere(), scope, engineType); FilterNode.getFilterField(sqlNode, whereFields); } return isAllMatch; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DimensionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DimensionNode.java index deac34f8f..dbfbb8d32 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DimensionNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DimensionNode.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; -import java.util.Arrays; import java.util.List; import java.util.Objects; import org.apache.calcite.sql.SqlNode; @@ -11,55 +11,47 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; public class DimensionNode extends SemanticNode { - public static SqlNode build(Dimension dimension, SqlValidatorScope scope) throws Exception { - SqlNode sqlNode = parse(dimension.getExpr(), scope); + public static SqlNode build(Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); return buildAs(dimension.getName(), sqlNode); } - public static List expand(Dimension dimension, SqlValidatorScope scope) throws Exception { - SqlNode sqlNode = parse(dimension.getExpr(), scope); + public static List expand(Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { + SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); return expand(sqlNode, scope); } - public static SqlNode buildName(Dimension dimension, SqlValidatorScope scope) throws Exception { - return parse(dimension.getName(), scope); + public static SqlNode buildName(Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { + return parse(dimension.getName(), scope, engineType); } - public static SqlNode buildExp(Dimension dimension, SqlValidatorScope scope) throws Exception { - return parse(dimension.getExpr(), scope); + public static SqlNode buildExp(Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { + return parse(dimension.getExpr(), scope, engineType); } - public static SqlNode buildNameAs(String alias, Dimension dimension, SqlValidatorScope scope) throws Exception { + public static SqlNode buildNameAs(String alias, Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { if ("".equals(alias)) { - return buildName(dimension, scope); + return buildName(dimension, scope, engineType); } - SqlNode sqlNode = parse(dimension.getName(), scope); + SqlNode sqlNode = parse(dimension.getName(), scope, engineType); return buildAs(alias, sqlNode); } - public static SqlNode buildArray(Dimension dimension, SqlValidatorScope scope) throws Exception { + public static SqlNode buildArray(Dimension dimension, SqlValidatorScope scope, EngineType engineType) + throws Exception { if (Objects.nonNull(dimension.getDataType()) && dimension.getDataType().isArray()) { - SqlNode sqlNode = parse(dimension.getExpr(), scope); + SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); if (isIdentifier(sqlNode)) { return buildAs(dimension.getName(), - parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope)); + parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope, engineType)); } throw new Exception("array dimension expr should only identify"); } - return build(dimension, scope); - } - - public static List expandArray(Dimension dimension, SqlValidatorScope scope) - throws Exception { - if (dimension.getDataType().isArray()) { - SqlNode sqlNode = parse(dimension.getExpr(), scope); - if (isIdentifier(sqlNode)) { - return Arrays.asList(buildAs(dimension.getName(), - parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope))); - } - throw new Exception("array dimension expr should only identify"); - } - return expand(dimension, scope); + return build(dimension, scope, engineType); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/IdentifyNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/IdentifyNode.java index e3d37faee..51f6028a4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/IdentifyNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/IdentifyNode.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify.Type; import java.util.List; @@ -11,8 +12,8 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; public class IdentifyNode extends SemanticNode { - public static SqlNode build(Identify identify, SqlValidatorScope scope) throws Exception { - return parse(identify.getName(), scope); + public static SqlNode build(Identify identify, SqlValidatorScope scope, EngineType engineType) throws Exception { + return parse(identify.getName(), scope, engineType); } public static Set getIdentifyNames(List identifies, Identify.Type type) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MeasureNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MeasureNode.java index 58ede5ffb..853cafe59 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MeasureNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MeasureNode.java @@ -1,50 +1,33 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; public class MeasureNode extends SemanticNode { - public static SqlNode build(Measure measure, boolean noAgg, SqlValidatorScope scope) throws Exception { - boolean addAgg = false; - if (!noAgg && measure.getAgg() != null && !measure.getAgg().isEmpty()) { - addAgg = true; - } - if (measure.getExpr() == null) { - if (addAgg) { - return parse(measure.getAgg() + " ( " + measure.getName() + " ) ", scope); - } - return parse(measure.getName(), scope); - } else { - if (addAgg) { - return buildAs(measure.getName(), parse(measure.getAgg() + " ( " + measure.getExpr() + " ) ", scope)); - } - return buildAs(measure.getName(), parse(measure.getExpr(), scope)); - } + public static SqlNode buildNonAgg(String alias, Measure measure, SqlValidatorScope scope, EngineType engineType) + throws Exception { + return buildAs(measure.getName(), getExpr(measure, alias, scope, engineType)); } - public static SqlNode buildNonAgg(String alias, Measure measure, SqlValidatorScope scope) throws Exception { - return buildAs(measure.getName(), getExpr(measure, alias, scope)); - } - - public static SqlNode buildAgg(Measure measure, boolean noAgg, SqlValidatorScope scope) throws Exception { + public static SqlNode buildAgg(Measure measure, boolean noAgg, SqlValidatorScope scope, EngineType engineType) + throws Exception { if ((measure.getAgg() == null || measure.getAgg().isEmpty()) || noAgg) { - return parse(measure.getName(), scope); + return parse(measure.getName(), scope, engineType); } - return buildAs(measure.getName(), AggFunctionNode.build(measure.getAgg(), measure.getName(), scope)); + return buildAs(measure.getName(), + AggFunctionNode.build(measure.getAgg(), measure.getName(), scope, engineType)); } - public static SqlNode buildAggAs(String aggFunc, String name, SqlValidatorScope scope) throws Exception { - return buildAs(name, AggFunctionNode.build(aggFunc, name, scope)); - } - - private static SqlNode getExpr(Measure measure, String alias, SqlValidatorScope scope) throws Exception { + private static SqlNode getExpr(Measure measure, String alias, SqlValidatorScope scope, EngineType enginType) + throws Exception { if (measure.getExpr() == null) { - return parse((alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope); + return parse((alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope, enginType); } - return parse(measure.getExpr(), scope); + return parse(measure.getExpr(), scope, enginType); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MetricNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MetricNode.java index 1b5736634..03b89e97a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MetricNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/MetricNode.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import java.util.HashMap; @@ -19,12 +20,12 @@ public class MetricNode extends SemanticNode { private Map measureFilter = new HashMap<>(); private Map aggFunction = new HashMap<>(); - public static SqlNode build(Metric metric, SqlValidatorScope scope) throws Exception { + public static SqlNode build(Metric metric, SqlValidatorScope scope, EngineType engineType) throws Exception { if (metric.getMetricTypeParams() == null || metric.getMetricTypeParams().getExpr() == null || metric.getMetricTypeParams().getExpr().isEmpty()) { - return parse(metric.getName(), scope); + return parse(metric.getName(), scope, engineType); } - SqlNode sqlNode = parse(metric.getMetricTypeParams().getExpr(), scope); + SqlNode sqlNode = parse(metric.getMetricTypeParams().getExpr(), scope, engineType); return buildAs(metric.getName(), sqlNode); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java index f476a8214..03bfbb000 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule; +import com.tencent.supersonic.headless.core.utils.SqlDialectFactory; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -18,7 +20,6 @@ import java.util.function.UnaryOperator; import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgramBuilder; @@ -40,7 +41,6 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlWith; import org.apache.calcite.sql.SqlWriterConfig; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.pretty.SqlPrettyWriter; @@ -74,8 +74,8 @@ public abstract class SemanticNode { AGGREGATION_FUNC.add("min"); } - public static SqlNode parse(String expression, SqlValidatorScope scope) throws Exception { - SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig()); + public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseExpression(); scope.validateExpr(sqlNode); return sqlNode; @@ -88,8 +88,9 @@ public abstract class SemanticNode { SqlParserPos.ZERO); } - public static String getSql(SqlNode sqlNode) { - SqlWriterConfig config = SqlPrettyWriter.config().withDialect(SemanticSqlDialect.DEFAULT) + public static String getSql(SqlNode sqlNode, EngineType engineType) { + SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); + SqlWriterConfig config = SqlPrettyWriter.config().withDialect(sqlDialect) .withKeywordsLowerCase(true).withClauseEndsLine(true).withAlwaysUseParentheses(false) .withSelectListItemsOnSeparateLines(false).withUpdateSetListNewline(false).withIndentation(0); @@ -390,20 +391,22 @@ public abstract class SemanticNode { return parseInfo; } - public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode) { + public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode, + EngineType engineType) { try { HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); - + SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); hepProgramBuilder.addRuleInstance(new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema)); RelOptPlanner relOptPlanner = new HepPlanner(hepProgramBuilder.build()); - RelToSqlConverter converter = new RelToSqlConverter(SemanticSqlDialect.DEFAULT); + RelToSqlConverter converter = new RelToSqlConverter(sqlDialect); SqlValidator sqlValidator = Configuration.getSqlValidator( - scope.getValidator().getCatalogReader().getRootSchema()); + scope.getValidator().getCatalogReader().getRootSchema(), engineType); SqlToRelConverter sqlToRelConverter = Configuration.getSqlToRelConverter(scope, sqlValidator, - relOptPlanner); + relOptPlanner, engineType); RelNode sqlRel = sqlToRelConverter.convertQuery( sqlValidator.validate(sqlNode), false, true).rel; - log.debug("RelNode optimize {}", SemanticNode.getSql(converter.visitRoot(sqlRel).asStatement())); + log.debug("RelNode optimize {}", + SemanticNode.getSql(converter.visitRoot(sqlRel).asStatement(), engineType)); relOptPlanner.setRoot(sqlRel); RelNode relNode = relOptPlanner.findBestExp(); return converter.visitRoot(relNode).asStatement(); @@ -413,13 +416,6 @@ public abstract class SemanticNode { return null; } - public static RelNode getRelNode(CalciteSchema rootSchema, SqlToRelConverter sqlToRelConverter, String sql) - throws SqlParseException { - SqlValidator sqlValidator = Configuration.getSqlValidator(rootSchema); - return sqlToRelConverter.convertQuery( - sqlValidator.validate(SqlParser.create(sql, SqlParser.Config.DEFAULT).parseStmt()), false, true).rel; - } - public static SqlBinaryOperator getBinaryOperator(String val) { if (val.equals("=")) { return SqlStdOperatorTable.EQUALS; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java index 84672e236..75f74b59e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; @@ -34,8 +35,10 @@ public class FilterRender extends Renderer { SqlNode filterNode = null; List queryMetrics = new ArrayList<>(metricCommand.getMetrics()); List queryDimensions = new ArrayList<>(metricCommand.getDimensions()); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + if (metricCommand.getWhere() != null && !metricCommand.getWhere().isEmpty()) { - filterNode = SemanticNode.parse(metricCommand.getWhere(), scope); + filterNode = SemanticNode.parse(metricCommand.getWhere(), scope, engineType); Set whereFields = new HashSet<>(); FilterNode.getFilterField(filterNode, whereFields); List fieldWhere = whereFields.stream().collect(Collectors.toList()); @@ -49,7 +52,7 @@ public class FilterRender extends Renderer { queryDimensions.addAll(dimensions); } for (String dimension : queryDimensions) { - tableView.getMeasure().add(SemanticNode.parse(dimension, scope)); + tableView.getMeasure().add(SemanticNode.parse(dimension, scope, engineType)); } for (String metric : queryMetrics) { Optional optionalMetric = Renderer.getMetricByName(metric, schema); @@ -58,9 +61,9 @@ public class FilterRender extends Renderer { continue; } if (optionalMetric.isPresent()) { - tableView.getMeasure().add(MetricNode.build(optionalMetric.get(), scope)); + tableView.getMeasure().add(MetricNode.build(optionalMetric.get(), scope, engineType)); } else { - tableView.getMeasure().add(SemanticNode.parse(metric, scope)); + tableView.getMeasure().add(SemanticNode.parse(metric, scope, engineType)); } } if (filterNode != null) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java index c77195258..63728bcf6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; @@ -49,13 +50,13 @@ public class JoinRender extends Renderer { @Override public void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); - //dataSources = getOrderSource(dataSources); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); if (queryWhere != null && !queryWhere.isEmpty()) { - SqlNode sqlNode = SemanticNode.parse(queryWhere, scope); + SqlNode sqlNode = SemanticNode.parse(queryWhere, scope, engineType); FilterNode.getFilterField(sqlNode, whereFields); fieldWhere = whereFields.stream().collect(Collectors.toList()); } @@ -129,9 +130,9 @@ public class JoinRender extends Renderer { if (!filterDimension.isEmpty()) { for (String d : getQueryDimension(filterDimension, queryAllDimension, whereFields)) { if (nonAgg) { - filterView.getMeasure().add(SemanticNode.parse(d, scope)); + filterView.getMeasure().add(SemanticNode.parse(d, scope, engineType)); } else { - filterView.getDimension().add(SemanticNode.parse(d, scope)); + filterView.getDimension().add(SemanticNode.parse(d, scope, engineType)); } } @@ -143,6 +144,7 @@ public class JoinRender extends Renderer { List reqMetrics, DataSource dataSource, Set sourceMeasure, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String m : reqMetrics) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { MetricNode metricNode = buildMetricNode(m, dataSource, scope, schema, nonAgg, alias); @@ -150,7 +152,8 @@ public class JoinRender extends Renderer { if (!metricNode.getNonAggNode().isEmpty()) { for (String measure : metricNode.getNonAggNode().keySet()) { innerSelect.put(measure, - SemanticNode.buildAs(measure, SemanticNode.parse(alias + "." + measure, scope))); + SemanticNode.buildAs(measure, + SemanticNode.parse(alias + "." + measure, scope, engineType))); } } @@ -159,10 +162,10 @@ public class JoinRender extends Renderer { if (metricNode.getNonAggNode().containsKey(entry.getKey())) { if (nonAgg) { filterView.getMeasure().add(SemanticNode.buildAs(entry.getKey(), - SemanticNode.parse(entry.getKey(), scope))); + SemanticNode.parse(entry.getKey(), scope, engineType))); } else { filterView.getMeasure().add(SemanticNode.buildAs(entry.getKey(), - AggFunctionNode.build(entry.getValue(), entry.getKey(), scope))); + AggFunctionNode.build(entry.getValue(), entry.getKey(), scope, engineType))); } } @@ -177,14 +180,16 @@ public class JoinRender extends Renderer { List reqDimensions, DataSource dataSource, Set dimension, SqlValidatorScope scope, SemanticSchema schema) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String d : reqDimensions) { if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) { String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY); innerSelect.put(d, - SemanticNode.buildAs(d, SemanticNode.parse(alias + "." + identifyDimension[1], scope))); + SemanticNode.buildAs(d, + SemanticNode.parse(alias + "." + identifyDimension[1], scope, engineType))); } else { - innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode.parse(alias + "." + d, scope))); + innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode.parse(alias + "." + d, scope, engineType))); } filterDimension.add(d); @@ -253,16 +258,16 @@ public class JoinRender extends Renderer { } private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map before, - DataSource dataSource, - SemanticSchema schema, SqlValidatorScope scope) + DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) throws Exception { - SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope, engineType); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema); SqlNode joinRelationCondition = null; if (!CollectionUtils.isEmpty(matchJoinRelation.getJoinCondition())) { sqlLiteral = SemanticNode.getJoinSqlLiteral(matchJoinRelation.getJoinType()); - joinRelationCondition = getCondition(matchJoinRelation, scope); + joinRelationCondition = getCondition(matchJoinRelation, scope, engineType); condition = joinRelationCondition; } if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) @@ -307,13 +312,13 @@ public class JoinRender extends Renderer { return matchJoinRelation; } - private SqlNode getCondition(JoinRelation joinRelation, - SqlValidatorScope scope) throws Exception { + private SqlNode getCondition(JoinRelation joinRelation, SqlValidatorScope scope, EngineType engineType) + throws Exception { SqlNode condition = null; for (Triple con : joinRelation.getJoinCondition()) { List ons = new ArrayList<>(); - ons.add(SemanticNode.parse(con.getLeft(), scope)); - ons.add(SemanticNode.parse(con.getRight(), scope)); + ons.add(SemanticNode.parse(con.getLeft(), scope, engineType)); + ons.add(SemanticNode.parse(con.getRight(), scope, engineType)); if (Objects.isNull(condition)) { condition = new SqlBasicCall( SemanticNode.getBinaryOperator(con.getMiddle()), @@ -334,7 +339,7 @@ public class JoinRender extends Renderer { } private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, - SqlValidatorScope scope) throws Exception { + SqlValidatorScope scope, EngineType engineType) throws Exception { Set selectLeft = SemanticNode.getSelect(left.getTable()); Set selectRight = SemanticNode.getSelect(right.getTable()); @@ -355,8 +360,8 @@ public class JoinRender extends Renderer { } } List ons = new ArrayList<>(); - ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope)); - ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope)); + ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope, engineType)); + ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope, engineType)); if (condition == null) { condition = new SqlBasicCall( SqlStdOperatorTable.EQUALS, @@ -454,18 +459,21 @@ public class JoinRender extends Renderer { endTime = zipper.getAlias() + "." + endTimeOp.get().getName(); dateTime = partMetric.getAlias() + "." + partTime.get().getName(); } - + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); + ArrayList operandList = new ArrayList<>( + Arrays.asList(SemanticNode.parse(endTime, scope, engineType), + SemanticNode.parse(dateTime, scope, engineType))); condition = new SqlBasicCall( SqlStdOperatorTable.AND, new ArrayList(Arrays.asList(new SqlBasicCall( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - new ArrayList(Arrays.asList(SemanticNode.parse(startTime, scope), - SemanticNode.parse(dateTime, scope))), + new ArrayList( + Arrays.asList(SemanticNode.parse(startTime, scope, engineType), + SemanticNode.parse(dateTime, scope, engineType))), SqlParserPos.ZERO, null), new SqlBasicCall( SqlStdOperatorTable.GREATER_THAN, - new ArrayList(Arrays.asList(SemanticNode.parse(endTime, scope), - SemanticNode.parse(dateTime, scope))), + operandList, SqlParserPos.ZERO, null))), SqlParserPos.ZERO, null); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java index 08e02b5ed..d64f79b6c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; -import com.tencent.supersonic.common.pojo.ColumnOrder; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; +import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; @@ -27,19 +28,20 @@ public class OutputRender extends Renderer { public void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { TableView selectDataSet = super.tableView; + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); for (String dimension : metricCommand.getDimensions()) { - selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope)); + selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope, engineType)); } for (String metric : metricCommand.getMetrics()) { if (MetricNode.isMetricField(metric, schema)) { // metric from field ignore continue; } - selectDataSet.getMeasure().add(SemanticNode.parse(metric, scope)); + selectDataSet.getMeasure().add(SemanticNode.parse(metric, scope, engineType)); } if (metricCommand.getLimit() > 0) { - SqlNode offset = SemanticNode.parse(metricCommand.getLimit().toString(), scope); + SqlNode offset = SemanticNode.parse(metricCommand.getLimit().toString(), scope, engineType); selectDataSet.setOffset(offset); } if (!CollectionUtils.isEmpty(metricCommand.getOrder())) { @@ -47,9 +49,9 @@ public class OutputRender extends Renderer { for (ColumnOrder columnOrder : metricCommand.getOrder()) { if (SqlStdOperatorTable.DESC.getName().equalsIgnoreCase(columnOrder.getOrder())) { orderList.add(SqlStdOperatorTable.DESC.createCall(SqlParserPos.ZERO, - new SqlNode[]{SemanticNode.parse(columnOrder.getCol(), scope)})); + new SqlNode[]{SemanticNode.parse(columnOrder.getCol(), scope, engineType)})); } else { - orderList.add(SemanticNode.parse(columnOrder.getCol(), scope)); + orderList.add(SemanticNode.parse(columnOrder.getCol(), scope, engineType)); } } selectDataSet.setOrder(new SqlNodeList(orderList, SqlParserPos.ZERO)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java index 118822404..d74331dab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; @@ -39,9 +40,9 @@ import java.util.stream.Collectors; public class SourceRender extends Renderer { public static TableView renderOne(String alias, List fieldWheres, - List reqMetrics, List reqDimensions, - String queryWhere, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + List reqMetrics, List reqDimensions, + String queryWhere, DataSource datasource, SqlValidatorScope scope, + SemanticSchema schema, boolean nonAgg) throws Exception { TableView dataSet = new TableView(); TableView output = new TableView(); @@ -97,24 +98,24 @@ public class SourceRender extends Renderer { boolean nonAgg, Set extendFields, TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception { List dimensionList = schema.getDimension().get(datasource.getName()); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); boolean isAdd = false; if (!CollectionUtils.isEmpty(dimensionList)) { for (Dimension dim : dimensionList) { if (!dim.getName().equalsIgnoreCase(dimension)) { continue; } - dataSet.getMeasure().add(DimensionNode.build(dim, scope)); + dataSet.getMeasure().add(DimensionNode.build(dim, scope, engineType)); if (nonAgg) { - //dataSet.getMeasure().addAll(DimensionNode.expand(dim, scope)); - output.getMeasure().add(DimensionNode.buildName(dim, scope)); + output.getMeasure().add(DimensionNode.buildName(dim, scope, engineType)); isAdd = true; continue; } if ("".equals(alias)) { - output.getDimension().add(DimensionNode.buildName(dim, scope)); + output.getDimension().add(DimensionNode.buildName(dim, scope, engineType)); } else { - output.getDimension().add(DimensionNode.buildNameAs(alias, dim, scope)); + output.getDimension().add(DimensionNode.buildNameAs(alias, dim, scope, engineType)); } isAdd = true; break; @@ -125,11 +126,11 @@ public class SourceRender extends Renderer { .filter(i -> i.getName().equalsIgnoreCase(dimension)).findFirst(); if (identify.isPresent()) { if (nonAgg) { - dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope)); - output.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope)); + dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + output.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); } else { - dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope)); - output.getDimension().add(SemanticNode.parse(identify.get().getName(), scope)); + dataSet.getMeasure().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); + output.getDimension().add(SemanticNode.parse(identify.get().getName(), scope, engineType)); } isAdd = true; } @@ -139,15 +140,15 @@ public class SourceRender extends Renderer { } Optional dimensionOptional = getDimensionByName(dimension, datasource); if (dimensionOptional.isPresent()) { - dataSet.getMeasure().add(DimensionNode.buildArray(dimensionOptional.get(), scope)); + dataSet.getMeasure().add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); if (dimensionOptional.get().getDataType().isArray()) { extendFields.add(dimensionOptional.get().getExpr()); } if (nonAgg) { - output.getMeasure().add(DimensionNode.buildName(dimensionOptional.get(), scope)); + output.getMeasure().add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); return; } - output.getDimension().add(DimensionNode.buildName(dimensionOptional.get(), scope)); + output.getDimension().add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); } } @@ -161,10 +162,10 @@ public class SourceRender extends Renderer { private static List getWhereMeasure(List fields, List queryMetrics, List queryDimensions, Set extendFields, DataSource datasource, SqlValidatorScope scope, - SemanticSchema schema, - boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); while (iterator.hasNext()) { String cur = iterator.next(); if (queryDimensions.contains(cur) || queryMetrics.contains(cur)) { @@ -179,13 +180,13 @@ public class SourceRender extends Renderer { if (!dim.getName().equalsIgnoreCase(where)) { continue; } - whereNode.addAll(DimensionNode.expand(dim, scope)); + whereNode.addAll(DimensionNode.expand(dim, scope, engineType)); isAdd = true; } } Optional identify = getIdentifyByName(where, datasource); if (identify.isPresent()) { - whereNode.add(IdentifyNode.build(identify.get(), scope)); + whereNode.add(IdentifyNode.build(identify.get(), scope, engineType)); isAdd = true; } if (isAdd) { @@ -193,7 +194,7 @@ public class SourceRender extends Renderer { } Optional dimensionOptional = getDimensionByName(where, datasource); if (dimensionOptional.isPresent()) { - whereNode.add(DimensionNode.buildArray(dimensionOptional.get(), scope)); + whereNode.add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); if (dimensionOptional.get().getDataType().isArray()) { extendFields.add(dimensionOptional.get().getExpr()); } @@ -317,12 +318,13 @@ public class SourceRender extends Renderer { } public void render(MetricQueryReq metricQueryReq, List dataSources, SqlValidatorScope scope, - SemanticSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricQueryReq.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); + EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); if (queryWhere != null && !queryWhere.isEmpty()) { - SqlNode sqlNode = SemanticNode.parse(queryWhere, scope); + SqlNode sqlNode = SemanticNode.parse(queryWhere, scope, engineType); FilterNode.getFilterField(sqlNode, whereFields); fieldWhere = whereFields.stream().collect(Collectors.toList()); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index e99bd8325..2afc17e0d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -109,7 +109,7 @@ public class CalculateAggConverter implements HeadlessConverter { ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); Database database = queryStatement.getSemanticModel().getDatabase(); ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, - EngineType.valueOf(database.getType().toUpperCase()), database.getVersion()); + EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setTables(parseSqlReq.getTables()); sqlCommend.setRootPath(parseSqlReq.getRootPath()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java new file mode 100644 index 000000000..26a354f3c --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java @@ -0,0 +1,48 @@ +package com.tencent.supersonic.headless.core.utils; + +import com.tencent.supersonic.headless.api.enums.EngineType; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlDialect.Context; +import org.apache.calcite.sql.SqlDialect.DatabaseProduct; + + +public class SqlDialectFactory { + + public static final Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.BIG_QUERY) + .withLiteralQuoteString("'") + .withLiteralEscapedQuoteString("''") + .withIdentifierQuoteString("`") + .withUnquotedCasing(Casing.UNCHANGED) + .withQuotedCasing(Casing.UNCHANGED) + .withCaseSensitive(false); + public static final Context POSTGRESQL_CONTEXT = SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.BIG_QUERY) + .withLiteralQuoteString("'") + .withLiteralEscapedQuoteString("''") + .withUnquotedCasing(Casing.UNCHANGED) + .withQuotedCasing(Casing.UNCHANGED) + .withCaseSensitive(false); + private static Map sqlDialectMap; + + static { + sqlDialectMap = new HashMap<>(); + sqlDialectMap.put(EngineType.CLICKHOUSE, new SemanticSqlDialect(DEFAULT_CONTEXT)); + sqlDialectMap.put(EngineType.MYSQL, new SemanticSqlDialect(DEFAULT_CONTEXT)); + sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT)); + sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT)); + } + + public static SemanticSqlDialect getSqlDialect(EngineType engineType) { + SemanticSqlDialect semanticSqlDialect = sqlDialectMap.get(engineType); + if (Objects.isNull(semanticSqlDialect)) { + return new SemanticSqlDialect(DEFAULT_CONTEXT); + } + return semanticSqlDialect; + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java index fc076b492..7846f0115 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java @@ -14,17 +14,11 @@ public class PostgresqlParametersBuilder implements DbParametersBuilder { public List build() { List databaseParameters = new ArrayList<>(); DatabaseParameter host = new DatabaseParameter(); - host.setComment("host"); - host.setName("host"); - host.setPlaceholder("请输入host"); + host.setComment("链接"); + host.setName("url"); + host.setPlaceholder("请输入链接"); databaseParameters.add(host); - DatabaseParameter port = new DatabaseParameter(); - port.setComment("port"); - port.setName("port"); - port.setPlaceholder("请输入端口号"); - databaseParameters.add(port); - DatabaseParameter userName = new DatabaseParameter(); userName.setComment("用户名"); userName.setName("username"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index b5f9782a1..6eaf44af0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -60,7 +60,6 @@ public class QueryController { User user = UserHolder.findUser(request, response); QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq); return queryService.queryBySql(querySqlReq, user); - //return queryService.queryByStructWithAuth(queryStructReq, user); } @PostMapping("/queryMetricDataById") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index 3eb0bcf49..ceca84b7c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -21,7 +21,7 @@ import java.util.List; public interface QueryService { - Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception; + SemanticQueryResp queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception; SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index ec8fe9510..2696ef64d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -44,7 +44,6 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission; import com.tencent.supersonic.headless.server.annotation.StructDataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; -import com.tencent.supersonic.headless.server.cache.CacheManager; import com.tencent.supersonic.headless.server.cache.QueryCache; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; @@ -67,7 +66,6 @@ import javax.servlet.http.HttpServletRequest; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -79,17 +77,11 @@ public class QueryServiceImpl implements QueryService { CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); private final StatUtils statUtils; - private final CacheManager cacheManager; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; private final Catalog catalog; private final AppService appService; - - @Value("${query.cache.enable:true}") - private Boolean cacheEnable; - private final QueryCache queryCache; - private final SemanticSchemaManager semanticSchemaManager; private final QueryParser queryParser; @@ -98,7 +90,6 @@ public class QueryServiceImpl implements QueryService { public QueryServiceImpl( StatUtils statUtils, - CacheManager cacheManager, QueryUtils queryUtils, QueryReqConverter queryReqConverter, Catalog catalog, @@ -108,7 +99,6 @@ public class QueryServiceImpl implements QueryService { QueryParser queryParser, QueryPlanner queryPlanner) { this.statUtils = statUtils; - this.cacheManager = cacheManager; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; this.catalog = catalog; @@ -122,10 +112,7 @@ public class QueryServiceImpl implements QueryService { @Override @S2SQLDataPermission @SneakyThrows - public Object queryBySql(QuerySqlReq querySQLReq, User user) { - statUtils.initStatInfo(querySQLReq, user); - QueryStatement queryStatement = new QueryStatement(); - SemanticQueryResp results = null; + public SemanticQueryResp queryBySql(QuerySqlReq querySQLReq, User user) { TaskStatusEnum state = TaskStatusEnum.SUCCESS; try { //1.initStatInfo @@ -137,23 +124,25 @@ public class QueryServiceImpl implements QueryService { } StatUtils.get().setUseResultCache(false); //3 query from db - queryStatement = convertToQueryStatement(querySQLReq, user); + QueryStatement queryStatement = convertToQueryStatement(querySQLReq, user); log.info("queryStatement:{}", queryStatement); - results = query(queryStatement); + SemanticQueryResp result = query(queryStatement); //4 reset cache and set stateInfo - Boolean setCacheSuccess = queryCache.put(querySQLReq, results); + Boolean setCacheSuccess = queryCache.put(querySQLReq, result); if (setCacheSuccess) { // if semanticQueryResp is not null, update cache data statUtils.updateResultCacheKey(queryCache.getCacheKey(querySQLReq)); } - if (Objects.isNull(results)) { + if (Objects.isNull(result)) { state = TaskStatusEnum.ERROR; } + return result; } catch (Exception e) { log.info("convertToQueryStatement has a exception:", e); + throw e; + } finally { + statUtils.statInfo2DbAsync(state); } - statUtils.statInfo2DbAsync(state); - return results; } public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) { @@ -186,7 +175,6 @@ public class QueryServiceImpl implements QueryService { @Override public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception { - SemanticQueryResp semanticQueryResp = null; TaskStatusEnum state = TaskStatusEnum.SUCCESS; log.info("[queryStructReq:{}]", queryStructReq); try { @@ -200,17 +188,17 @@ public class QueryServiceImpl implements QueryService { StatUtils.get().setUseResultCache(false); //3 query QueryStatement queryStatement = buildQueryStatement(queryStructReq); - semanticQueryResp = query(queryStatement); + SemanticQueryResp result = query(queryStatement); //4 reset cache and set stateInfo - Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp); + Boolean setCacheSuccess = queryCache.put(queryStructReq, result); if (setCacheSuccess) { - // if semanticQueryResp is not null, update cache data + // if result is not null, update cache data statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq)); } - if (Objects.isNull(semanticQueryResp)) { + if (Objects.isNull(result)) { state = TaskStatusEnum.ERROR; } - return semanticQueryResp; + return result; } catch (Exception e) { log.error("exception in queryByStruct, e: ", e); state = TaskStatusEnum.ERROR; @@ -447,28 +435,6 @@ public class QueryServiceImpl implements QueryService { return ExplainResp.builder().sql(sql).build(); } - private boolean isCache(QueryMultiStructReq queryStructReq) { - if (!cacheEnable) { - return false; - } - if (!CollectionUtils.isEmpty(queryStructReq.getQueryStructReqs()) - && queryStructReq.getQueryStructReqs().get(0).getCacheInfo() != null) { - return queryStructReq.getQueryStructReqs().get(0).getCacheInfo().getCache(); - } - return false; - } - - private SemanticQueryResp queryByCache(String key, Object queryCmd) { - - Object resultObject = cacheManager.get(key); - if (Objects.nonNull(resultObject)) { - log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString()); - statUtils.updateResultCacheKey(key); - return (SemanticQueryResp) resultObject; - } - return null; - } - private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { QuerySqlReq querySQLReq = new QuerySqlReq(); List modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 167133216..9c0bf8823 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -115,7 +115,7 @@ public class QueryReqConverter { result.setRootPath(querySQLReq.getModelIdStr()); result.setTables(tables); DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0)); - if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()), + if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 7c24128ca..c942db3a0 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.calcite; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.response.SqlParserResp; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; @@ -43,7 +44,8 @@ class HeadlessParserServiceTest { QueryStatement queryStatement = new QueryStatement(); queryStatement.setMetricReq(metricCommand); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); - sqlParser.setSql(aggBuilder.getSql()); + EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); + sqlParser.setSql(aggBuilder.getSql(engineType)); sqlParser.setSourceId(aggBuilder.getSourceId()); } catch (Exception e) { sqlParser.setErrMsg(e.getMessage());