diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index 3cc5eb3b4..afa4001fe 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -5,8 +5,8 @@ import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; -import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; +import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -26,13 +26,13 @@ public class DefaultSemanticTranslator implements SemanticTranslator { return; } try { - for (QueryConverter converter : ComponentFactory.getQueryConverters()) { - if (converter.accept(queryStatement)) { - log.debug("QueryConverter accept [{}]", converter.getClass().getName()); - converter.convert(queryStatement); + for (QueryParser parser : ComponentFactory.getQueryParsers()) { + if (parser.accept(queryStatement)) { + log.debug("QueryConverter accept [{}]", parser.getClass().getName()); + parser.parse(queryStatement); } } - doOntologyParse(queryStatement); + mergeOntologyQuery(queryStatement); if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) { queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql()); @@ -41,8 +41,10 @@ public class DefaultSemanticTranslator implements SemanticTranslator { throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); } - for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { - queryOptimizer.rewrite(queryStatement); + for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) { + if (optimizer.accept(queryStatement)) { + optimizer.rewrite(queryStatement); + } } log.info("translated query SQL: [{}]", queryStatement.getSql()); } catch (Exception e) { @@ -51,10 +53,10 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } } - private void doOntologyParse(QueryStatement queryStatement) throws Exception { + private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); log.info("parse with ontology: [{}]", ontologyQuery); - ComponentFactory.getQueryParser().parse(queryStatement); + if (!queryStatement.isOk()) { throw new Exception(String.format("parse ontology table [%s] error [%s]", queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); @@ -67,31 +69,30 @@ public class DefaultSemanticTranslator implements SemanticTranslator { List> tables = new ArrayList<>(); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); + String finalSql = null; if (sqlQuery.isSupportWith()) { EngineType engineType = EngineType.fromString(queryStatement.getOntology().getDatabase().getType()); if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { - String withSql = "with " + tables.stream() + finalSql = "with " + tables.stream() .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) .collect(Collectors.joining(",")) + "\n" + ontologyQuerySql; - queryStatement.setSql(withSql); } else { List withTableList = tables.stream().map(Pair::getLeft).collect(Collectors.toList()); List withSqlList = tables.stream().map(Pair::getRight).collect(Collectors.toList()); - String mergeSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, - withSqlList, withTableList); - queryStatement.setSql(mergeSql); + finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList, + withTableList); } } else { for (Pair tb : tables) { - ontologyQuerySql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), + finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()), -1); } - queryStatement.setSql(ontologyQuerySql); } + queryStatement.setSql(finalSql); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java deleted file mode 100644 index ed9d407f8..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.converter; - -import com.tencent.supersonic.headless.core.pojo.QueryStatement; - -/** - * A query converter performs preprocessing work to the QueryStatement before parsing. - */ -public interface QueryConverter { - - boolean accept(QueryStatement queryStatement); - - void convert(QueryStatement queryStatement) throws Exception; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java index 96d109439..c8ae12fb7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java @@ -14,16 +14,20 @@ import java.util.Objects; @Component("DbDialectOptimizer") public class DbDialectOptimizer implements QueryOptimizer { + @Override + public boolean accept(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + return Objects.nonNull(database) && Objects.nonNull(database.getType()); + } + @Override public void rewrite(QueryStatement queryStatement) { SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); DatabaseResp database = semanticSchemaResp.getDatabaseResp(); String sql = queryStatement.getSql(); - if (Objects.isNull(database) || Objects.isNull(database.getType())) { - return; - } - String type = database.getType(); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); + DbAdaptor engineAdaptor = + DbAdaptorFactory.getEngineAdaptor(database.getType().toLowerCase()); if (Objects.nonNull(engineAdaptor)) { String adaptedSql = engineAdaptor.rewriteSql(sql); queryStatement.setSql(adaptedSql); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java deleted file mode 100644 index 47a4b1bab..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.optimizer; - -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.StructQuery; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import java.util.Objects; - -/** Remove the default metric added by the system when the query only has dimensions */ -@Slf4j -@Component("DetailQueryOptimizer") -public class DetailQueryOptimizer implements QueryOptimizer { - - @Override - public void rewrite(QueryStatement queryStatement) { - StructQuery structQuery = queryStatement.getStructQuery(); - String sqlRaw = queryStatement.getSql().trim(); - if (StringUtils.isEmpty(sqlRaw)) { - throw new RuntimeException("sql is empty or null"); - } - log.debug("before handleNoMetric, sql:{}", sqlRaw); - // if (isDetailQuery(structQueryParam)) { - // if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) { - // String sqlForm = "select %s from ( %s ) src_no_metric"; - // String sql = String.format(sqlForm, - // structQueryParam.getGroups().stream().collect(Collectors.joining(",")), - // sqlRaw); - // queryStatement.setSql(sql); - // } - // } - log.debug("after handleNoMetric, sql:{}", queryStatement.getSql()); - } - - public boolean isDetailQuery(StructQuery structQuery) { - return Objects.nonNull(structQuery) && structQuery.getQueryType().isNativeAggQuery(); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java index baeb626dd..b8ac22bf1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java @@ -7,5 +7,9 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; * derive the most efficient query. */ public interface QueryOptimizer { + + boolean accept(QueryStatement queryStatement); + void rewrite(QueryStatement queryStatement); + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java index 3d13d2d0a..ed7d56fea 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java @@ -9,10 +9,13 @@ import org.springframework.stereotype.Component; @Component("ResultLimitOptimizer") public class ResultLimitOptimizer implements QueryOptimizer { + @Override + public boolean accept(QueryStatement queryStatement) { + return !SqlSelectHelper.hasLimit(queryStatement.getSql()); + } + @Override public void rewrite(QueryStatement queryStatement) { - if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) { - queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit()); - } + queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit()); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java similarity index 89% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java index 959d4869e..01d9f9299 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; @@ -22,11 +22,11 @@ import java.util.stream.Collectors; /** - * This converter appends default dimension values (if configured) to the where statement. + * This parser appends default dimension values (if configured) to the where statement. */ @Slf4j -@Component("DefaultDimValueConverter") -public class DefaultDimValueConverter implements QueryConverter { +@Component("DefaultDimValueParser") +public class DefaultDimValueParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { @@ -35,7 +35,7 @@ public class DefaultDimValueConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) { + public void parse(QueryStatement queryStatement) { List dimensions = queryStatement.getOntology().getDimensions().stream() .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .collect(Collectors.toList()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DimExpressionParser.java similarity index 70% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DimExpressionParser.java index da96e2d10..2cf058a1b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DimExpressionParser.java @@ -1,8 +1,7 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; @@ -13,15 +12,17 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; /** - * This converter replaces dimension bizName in the S2SQL with calculation expression (if - * configured). + * This parser replaces dimension bizName in the S2SQL with calculation expression (if configured). */ -@Component("DimExpressionConverter") +@Component("DimExpressionParser") @Slf4j -public class DimExpressionConverter implements QueryConverter { +public class DimExpressionParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { return Objects.nonNull(queryStatement.getSqlQuery()) @@ -29,7 +30,7 @@ public class DimExpressionConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema(); SqlQuery sqlQuery = queryStatement.getSqlQuery(); @@ -49,22 +50,11 @@ public class DimExpressionConverter implements QueryConverter { Set queryFields = ontologyQuery.getFields(); log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions); - Set allFields = new HashSet<>(); - Map dimensionMap = new HashMap<>(); - semanticSchema.getModelResps().forEach(modelResp -> { - allFields.addAll(modelResp.getFieldList()); - if (modelResp.getModelDetail().getDimensions() != null) { - modelResp.getModelDetail().getDimensions() - .forEach(dimension -> dimensionMap.put(dimension.getBizName(), dimension)); - } - }); - - Map dim2Expr = new HashMap<>(); for (DimSchemaResp queryDim : queryDimensions) { queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr())); - dim2Expr.put(queryDim.getBizName(), queryDim.getExpr()); queryFields.addAll(queryDim.getFields()); + dim2Expr.put(queryDim.getBizName(), queryDim.getExpr()); } return dim2Expr; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricExpressionParser.java similarity index 87% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricExpressionParser.java index 7d57529d3..deb9b4ed9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricExpressionParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; @@ -17,11 +17,11 @@ import org.springframework.util.CollectionUtils; import java.util.*; /** - * This converter replaces metric bizName in the S2SQL with calculation expression (if configured). + * This parser replaces metric bizName in the S2SQL with calculation expression (if configured). */ -@Component("MetricExpressionConverter") +@Component("MetricExpressionParser") @Slf4j -public class MetricExpressionConverter implements QueryConverter { +public class MetricExpressionParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { return Objects.nonNull(queryStatement.getSqlQuery()) @@ -29,7 +29,7 @@ public class MetricExpressionConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema(); SqlQuery sqlQuery = queryStatement.getSqlQuery(); @@ -64,10 +64,10 @@ public class MetricExpressionConverter implements QueryConverter { Map metric2Expr = new HashMap<>(); for (MetricSchemaResp queryMetric : queryMetrics) { String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(), - queryMetric.getMetricDefineType(), visitedMetrics, queryFields); + queryMetric.getMetricDefineType(), visitedMetrics); // add all fields referenced in the expression queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr)); - log.debug("derived metric {}->{}", queryMetric.getBizName(), fieldExpr); + queryFields.addAll(queryMetric.getFields()); if (!queryMetric.getBizName().equals(fieldExpr)) { metric2Expr.put(queryMetric.getBizName(), fieldExpr); } @@ -78,8 +78,7 @@ public class MetricExpressionConverter implements QueryConverter { private String buildFieldExpr(final List metricResps, final Map allMeasures, final String metricExpr, - final MetricDefineType metricDefineType, Map visitedMetric, - Set queryFields) { + final MetricDefineType metricDefineType, Map visitedMetric) { Set fields = SqlSelectHelper.getFieldsFromExpr(metricExpr); if (!CollectionUtils.isEmpty(fields)) { Map replace = new HashMap<>(); @@ -97,8 +96,7 @@ public class MetricExpressionConverter implements QueryConverter { replace.put(field, buildFieldExpr(metricResps, allMeasures, metricItem.get().getExpr(), - metricItem.get().getMetricDefineType(), visitedMetric, - queryFields)); + metricItem.get().getMetricDefineType(), visitedMetric)); visitedMetric.put(field, replace.get(field)); } break; @@ -111,19 +109,16 @@ public class MetricExpressionConverter implements QueryConverter { expr = String.format("%s (%s)", measure.getAgg(), metricExpr); } replace.put(field, expr); - queryFields.add(field); } break; case FIELD: - queryFields.add(field); - break; default: break; } } if (!CollectionUtils.isEmpty(replace)) { String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace); - log.debug("derived measure {}->{}", metricExpr, expr); + log.debug("derived metric {}->{}", metricExpr, expr); return expr; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java index 868d8c0d4..4e813bba1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; @@ -21,9 +21,9 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -@Component("MetricRatioConverter") +@Component("MetricRatioParser") @Slf4j -public class MetricRatioConverter implements QueryConverter { +public class MetricRatioParser implements QueryParser { public interface EngineSql { @@ -58,7 +58,7 @@ public class MetricRatioConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { DatabaseResp database = queryStatement.getOntology().getDatabase(); generateRatioSql(queryStatement, EngineType.fromString(database.getType()), database.getVersion()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java similarity index 59% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java index 89d4fffd5..e4467467a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java @@ -1,23 +1,30 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.parser.QueryParser; +import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -@Component("CalciteQueryParser") +import java.util.Objects; + +/** + * This parser generates inner sql statement for the ontology query, which would be selected by the + * parsed sql query. + */ +@Component("OntologyQueryParser") @Slf4j -public class CalciteQueryParser implements QueryParser { +public class OntologyQueryParser implements QueryParser { + + @Override + public boolean accept(QueryStatement queryStatement) { + return Objects.nonNull(queryStatement.getOntologyQuery()); + } @Override public void parse(QueryStatement queryStatement) throws Exception { Ontology ontology = queryStatement.getOntology(); - if (ontology == null) { - queryStatement.setErrMsg("No ontology could be found"); - return; - } - S2CalciteSchema semanticSchema = S2CalciteSchema.builder() .schemaKey("DATASET_" + queryStatement.getDataSetId()).ontology(ontology) .runtimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java index 90158664b..8e83b904e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java @@ -6,5 +6,8 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; * A query parser generates physical SQL for the QueryStatement. */ public interface QueryParser { + + boolean accept(QueryStatement queryStatement); + void parse(QueryStatement queryStatement) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/RuntimeOptions.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/RuntimeOptions.java similarity index 76% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/RuntimeOptions.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/RuntimeOptions.java index 67663d5b1..306ecb799 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/RuntimeOptions.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/RuntimeOptions.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite; +package com.tencent.supersonic.headless.core.translator.parser; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java similarity index 93% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index e03e03cc8..e782fb4ed 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; @@ -23,12 +23,12 @@ import java.util.Map; import java.util.Objects; /** - * This converter rewrites S2SQL including conversion from metric/dimension name to bizName and - * build ontology query in preparation for generation of physical SQL. + * This parser rewrites S2SQL including conversion from metric/dimension name to bizName and build + * ontology query in preparation for generation of physical SQL. */ -@Component("SqlQueryConverter") +@Component("SqlQueryParser") @Slf4j -public class SqlQueryConverter implements QueryConverter { +public class SqlQueryParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { @@ -36,7 +36,7 @@ public class SqlQueryConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { convertNameToBizName(queryStatement); rewriteOrderBy(queryStatement); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java similarity index 88% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java index 8a35c2b93..cffc67fcb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; @@ -13,8 +13,8 @@ import java.util.List; import java.util.Objects; @Slf4j -@Component("SqlVariableConverter") -public class SqlVariableConverter implements QueryConverter { +@Component("SqlVariableParser") +public class SqlVariableParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { @@ -22,7 +22,7 @@ public class SqlVariableConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) { + public void parse(QueryStatement queryStatement) { SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); List modelResps = semanticSchemaResp.getModelResps(); if (CollectionUtils.isEmpty(modelResps)) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java similarity index 86% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java index f84a263dd..8f386c01d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.ContextUtils; @@ -13,12 +13,12 @@ import org.springframework.stereotype.Component; import java.util.Objects; /** - * This converter converts struct semantic query into sql query by generating S2SQL based on - * structured semantic information. + * This parser converts struct semantic query into sql query by generating S2SQL based on structured + * semantic information. */ -@Component("StructQueryConverter") +@Component("StructQueryParser") @Slf4j -public class StructQueryConverter implements QueryConverter { +public class StructQueryParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { @@ -26,7 +26,7 @@ public class StructQueryConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); StructQuery structQuery = queryStatement.getStructQuery(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java index 128a4c1a2..0b1d80a18 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.pojo.Ontology; +import com.tencent.supersonic.headless.core.translator.parser.RuntimeOptions; import lombok.Builder; import lombok.Data; import org.apache.calcite.schema.Schema; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index b6e573e6d..c4531ffa8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryAccelerator; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import lombok.extern.slf4j.Slf4j; @@ -20,15 +19,13 @@ import java.util.stream.Collectors; @Slf4j public class ComponentFactory { - private static List queryConverters = new ArrayList<>(); private static Map queryOptimizers = new HashMap<>(); private static List queryExecutors = new ArrayList<>(); private static List queryAccelerators = new ArrayList<>(); - private static QueryParser queryParser; + private static List queryParsers = new ArrayList<>(); private static QueryCache queryCache; static { - initQueryConverter(); initQueryOptimizer(); initQueryExecutors(); initQueryAccelerators(); @@ -36,13 +33,6 @@ public class ComponentFactory { initQueryCache(); } - public static List getQueryConverters() { - if (queryConverters.isEmpty()) { - initQueryConverter(); - } - return queryConverters; - } - public static List getQueryOptimizers() { if (queryOptimizers.isEmpty()) { initQueryOptimizer(); @@ -64,11 +54,11 @@ public class ComponentFactory { return queryAccelerators; } - public static QueryParser getQueryParser() { - if (queryParser == null) { + public static List getQueryParsers() { + if (queryParsers == null) { initQueryParser(); } - return queryParser; + return queryParsers; } public static QueryCache getQueryCache() { @@ -103,12 +93,8 @@ public class ComponentFactory { init(QueryAccelerator.class, queryAccelerators); } - private static void initQueryConverter() { - init(QueryConverter.class, queryConverters); - } - private static void initQueryParser() { - queryParser = init(QueryParser.class); + init(QueryParser.class, queryParsers); } private static void initQueryCache() { diff --git a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java index c85fe7141..620b716e5 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.parser.aggregate; import com.alibaba.fastjson.JSON; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser; +import com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser; import org.junit.jupiter.api.Test; import org.testng.Assert; @@ -316,7 +316,7 @@ public class CalciteSqlParserTest { + " \"createdAt\": 1711367511146,\n" + " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}"; QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class); - CalciteQueryParser calciteSqlParser = new CalciteQueryParser(); + OntologyQueryParser calciteSqlParser = new OntologyQueryParser(); calciteSqlParser.parse(queryStatement); Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""), "SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`" diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index 12ac957a1..a2687a348 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -25,21 +25,19 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ ### headless-core SPIs -com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ - com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter - com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ - com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ - com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser + com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\ + com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\ + com.tencent.supersonic.headless.core.translator.parser.DimExpressionParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricExpressionParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\ + com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.JdbcExecutor diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 22845e747..8f7fce7d8 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -25,22 +25,19 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ ### headless-core SPIs -com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ - com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.DimExpressionConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter - com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ - com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ - com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser + com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\ + com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\ + com.tencent.supersonic.headless.core.translator.parser.DimExpressionParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricExpressionParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\ + com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.JdbcExecutor