From 4cb225635182acc1e943c34b24e6748b55fdad5b Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Thu, 19 Dec 2024 21:41:47 +0800 Subject: [PATCH] [improvement][headless]Merge function of QueryConverter abstraction to QueryParser. --- .../chat/corrector/LLMSqlCorrector.java | 2 +- .../parser/s2sql => pojo}/JoinRelation.java | 2 +- .../parser/s2sql => pojo}/Ontology.java | 7 +- .../OntologyQuery.java} | 4 +- .../headless/core/pojo/QueryStatement.java | 10 +- .../{SqlQueryParam.java => SqlQuery.java} | 2 +- ...StructQueryParam.java => StructQuery.java} | 2 +- .../translator/DefaultSemanticTranslator.java | 56 +++-- .../translator/converter/QueryConverter.java | 11 - .../converter/StructQueryConverter.java | 74 ------ .../optimizer/DbDialectOptimizer.java | 9 +- .../optimizer/DetailQueryOptimizer.java | 40 --- .../translator/optimizer/QueryOptimizer.java | 4 + .../optimizer/ResultLimitOptimizer.java | 9 +- .../DefaultDimValueParser.java} | 20 +- .../MetricRatioParser.java} | 234 +++++++++--------- ...ryParser.java => OntologyQueryParser.java} | 24 +- .../core/translator/parser/QueryParser.java | 3 + .../SqlQueryParser.java} | 84 +++---- .../SqlVariableParser.java} | 15 +- .../translator/parser/StructQueryParser.java | 72 ++++++ .../parser/calcite/S2CalciteSchema.java | 4 +- .../translator/parser/calcite/SqlBuilder.java | 21 +- .../parser/calcite/node/DataModelNode.java | 17 +- .../parser/calcite/node/SemanticNode.java | 2 +- .../parser/calcite/render/FilterRender.java | 4 +- .../parser/calcite/render/JoinRender.java | 6 +- .../parser/calcite/render/OutputRender.java | 4 +- .../parser/calcite/render/Renderer.java | 4 +- .../parser/calcite/render/SourceRender.java | 14 +- .../headless/core/utils/ComponentFactory.java | 34 +-- .../headless/core/utils/SqlGenerateUtils.java | 67 +++-- .../aggregate/CalciteSqlParserTest.java | 4 +- .../service/impl/S2SemanticLayerService.java | 21 +- .../server/manager/SemanticSchemaManager.java | 4 +- .../server/utils/MetricDrillDownChecker.java | 2 +- .../calcite/HeadlessParserServiceTest.java | 14 +- .../main/resources/META-INF/spring.factories | 17 +- .../main/resources/META-INF/spring.factories | 17 +- 39 files changed, 440 insertions(+), 500 deletions(-) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/{translator/parser/s2sql => pojo}/JoinRelation.java (82%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/{translator/parser/s2sql => pojo}/Ontology.java (66%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/{translator/parser/s2sql/OntologyQueryParam.java => pojo/OntologyQuery.java} (83%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/{SqlQueryParam.java => SqlQuery.java} (89%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/{StructQueryParam.java => StructQuery.java} (96%) delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java rename headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/{converter/DefaultDimValueConverter.java => parser/DefaultDimValueParser.java} (79%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/{converter/MetricRatioConverter.java => parser/MetricRatioParser.java} (56%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/{calcite/CalciteQueryParser.java => OntologyQueryParser.java} (57%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/{converter/SqlQueryConverter.java => parser/SqlQueryParser.java} (86%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/{converter/SqlVariableConverter.java => parser/SqlVariableParser.java} (77%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java index 368e2c05a..6452da92c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java @@ -65,7 +65,7 @@ public class LLMSqlCorrector extends BaseSemanticCorrector { return; } - Text2SQLExemplar exemplar = (Text2SQLExemplar)semanticParseInfo.getProperties() + Text2SQLExemplar exemplar = (Text2SQLExemplar) semanticParseInfo.getProperties() .get(Text2SQLExemplar.PROPERTY_KEY); ChatLanguageModel chatLanguageModel = diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/JoinRelation.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JoinRelation.java similarity index 82% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/JoinRelation.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JoinRelation.java index ebf7b828f..1ff1dc3fc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/JoinRelation.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JoinRelation.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.pojo; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Ontology.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java similarity index 66% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Ontology.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java index 64d802d51..daba74b36 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Ontology.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java @@ -1,6 +1,9 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.pojo; -import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; +import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization; +import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; import lombok.Data; import java.util.ArrayList; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java similarity index 83% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java index 48b9a47ab..4ee7c49c1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/OntologyQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.pojo; import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.ColumnOrder; @@ -9,7 +9,7 @@ import java.util.List; import java.util.Set; @Data -public class OntologyQueryParam { +public class OntologyQuery { private Set metrics = Sets.newHashSet(); private Set dimensions = Sets.newHashSet(); private String where; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 8360ec944..59058fff2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -1,8 +1,6 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Ontology; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; @@ -13,15 +11,15 @@ public class QueryStatement { private Long dataSetId; private String sql; private String errMsg; - private StructQueryParam structQueryParam; - private SqlQueryParam sqlQueryParam; - private OntologyQueryParam ontologyQueryParam; + private StructQuery structQuery; + private SqlQuery sqlQuery; + private OntologyQuery ontologyQuery; private Integer status = 0; private Boolean isS2SQL = false; private Boolean enableOptimize = true; private Triple minMaxTime; private Ontology ontology; - private SemanticSchemaResp semanticSchemaResp; + private SemanticSchemaResp semanticSchema; private Integer limit = 1000; private Boolean isTranslated = false; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQuery.java similarity index 89% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQuery.java index 086397e88..09da3ea05 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQuery.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.pojo; import lombok.Data; @Data -public class SqlQueryParam { +public class SqlQuery { private String sql; private String table; private boolean supportWith = true; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java index 8fd129885..f6e9489ef 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQuery.java @@ -12,7 +12,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class StructQueryParam { +public class StructQuery { private List groups = new ArrayList(); private List aggregators = new ArrayList(); private List orders = new ArrayList(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index c163a9a4a..e931f8801 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -3,10 +3,10 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; -import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.translator.parser.QueryParser; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -26,23 +26,30 @@ public class DefaultSemanticTranslator implements SemanticTranslator { return; } try { - for (QueryConverter converter : ComponentFactory.getQueryConverters()) { - if (converter.accept(queryStatement)) { - log.debug("QueryConverter accept [{}]", converter.getClass().getName()); - converter.convert(queryStatement); + for (QueryParser parser : ComponentFactory.getQueryParser()) { + if (parser.accept(queryStatement)) { + log.debug("QueryConverter accept [{}]", parser.getClass().getName()); + parser.parse(queryStatement); } } - doOntologyParse(queryStatement); + if (!queryStatement.isOk()) { + throw new Exception(String.format("parse ontology table [%s] error [%s]", + queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); + } - if (StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSimplifiedSql())) { - queryStatement.setSql(queryStatement.getSqlQueryParam().getSimplifiedSql()); + mergeOntologyQuery(queryStatement); + + if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) { + queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql()); } if (StringUtils.isBlank(queryStatement.getSql())) { throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); } - for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { - queryOptimizer.rewrite(queryStatement); + for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) { + if (optimizer.accept(queryStatement)) { + optimizer.rewrite(queryStatement); + } } log.info("translated query SQL: [{}]", StringUtils.normalizeSpace(queryStatement.getSql())); @@ -52,23 +59,18 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } } - private void doOntologyParse(QueryStatement queryStatement) throws Exception { - OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam(); - log.info("parse with ontology: [{}]", ontologyQueryParam); - ComponentFactory.getQueryParser().parse(queryStatement); - if (!queryStatement.isOk()) { - throw new Exception(String.format("parse ontology table [%s] error [%s]", - queryStatement.getSqlQueryParam().getTable(), queryStatement.getErrMsg())); - } + private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { + OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); + log.info("parse with ontology: [{}]", ontologyQuery); - SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam(); - String ontologyQuerySql = sqlQueryParam.getSql(); - String ontologyInnerTable = sqlQueryParam.getTable(); + SqlQuery sqlQuery = queryStatement.getSqlQuery(); + String ontologyQuerySql = sqlQuery.getSql(); + String ontologyInnerTable = sqlQuery.getTable(); String ontologyInnerSql = queryStatement.getSql(); List> tables = new ArrayList<>(); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); - if (sqlQueryParam.isSupportWith()) { + if (sqlQuery.isSupportWith()) { EngineType engineType = queryStatement.getOntology().getDatabase().getType(); if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { String withSql = "with " + tables.stream() @@ -86,9 +88,9 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } } else { for (Pair tb : tables) { - ontologyQuerySql = - StringUtils.replace(ontologyQuerySql, tb.getLeft(), "(" + tb.getRight() - + ") " + (sqlQueryParam.isWithAlias() ? "" : tb.getLeft()), -1); + ontologyQuerySql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), + "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()), + -1); } queryStatement.setSql(ontologyQuerySql); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java deleted file mode 100644 index cb2b46bd1..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/QueryConverter.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.converter; - -import com.tencent.supersonic.headless.core.pojo.QueryStatement; - -/** to supplement,translate the request Body */ -public interface QueryConverter { - - boolean accept(QueryStatement queryStatement); - - void convert(QueryStatement queryStatement) throws Exception; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java deleted file mode 100644 index c0c3d79e6..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.converter; - -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.pojo.Database; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; -import com.tencent.supersonic.headless.core.pojo.StructQueryParam; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; -import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.util.Objects; -import java.util.stream.Collectors; - -@Component("ParserDefaultConverter") -@Slf4j -public class StructQueryConverter implements QueryConverter { - - @Override - public boolean accept(QueryStatement queryStatement) { - return Objects.nonNull(queryStatement.getStructQueryParam()) - && !queryStatement.getIsS2SQL(); - } - - @Override - public void convert(QueryStatement queryStatement) throws Exception { - SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); - - String dsTable = "t_1"; - SqlQueryParam sqlParam = new SqlQueryParam(); - sqlParam.setTable(dsTable); - String sql = String.format("select %s from %s %s %s %s", - sqlGenerateUtils.getSelect(structQueryParam), dsTable, - sqlGenerateUtils.getGroupBy(structQueryParam), - sqlGenerateUtils.getOrderBy(structQueryParam), - sqlGenerateUtils.getLimit(structQueryParam)); - Database database = queryStatement.getOntology().getDatabase(); - if (!sqlGenerateUtils.isSupportWith(database.getType(), database.getVersion())) { - sqlParam.setSupportWith(false); - sql = String.format("select %s from %s t0 %s %s %s", - sqlGenerateUtils.getSelect(structQueryParam), dsTable, - sqlGenerateUtils.getGroupBy(structQueryParam), - sqlGenerateUtils.getOrderBy(structQueryParam), - sqlGenerateUtils.getLimit(structQueryParam)); - } - sqlParam.setSql(sql); - queryStatement.setSqlQueryParam(sqlParam); - - OntologyQueryParam ontologyQueryParam = new OntologyQueryParam(); - ontologyQueryParam.getDimensions().addAll(structQueryParam.getGroups()); - ontologyQueryParam.getMetrics().addAll(structQueryParam.getAggregators().stream() - .map(Aggregator::getColumn).collect(Collectors.toList())); - String where = sqlGenerateUtils.generateWhere(structQueryParam, null); - ontologyQueryParam.setWhere(where); - if (ontologyQueryParam.getMetrics().isEmpty()) { - ontologyQueryParam.setAggOption(AggOption.NATIVE); - } else { - ontologyQueryParam.setAggOption(AggOption.DEFAULT); - } - ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery()); - ontologyQueryParam.setOrder(structQueryParam.getOrders().stream() - .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) - .collect(Collectors.toList())); - ontologyQueryParam.setLimit(structQueryParam.getLimit()); - queryStatement.setOntologyQueryParam(ontologyQueryParam); - log.info("parse structQuery [{}] ", queryStatement.getSqlQueryParam()); - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java index 481379e28..cc89ab0fb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DbDialectOptimizer.java @@ -14,9 +14,16 @@ import java.util.Objects; @Component("DbDialectOptimizer") public class DbDialectOptimizer implements QueryOptimizer { + @Override + public boolean accept(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + return Objects.nonNull(database) && Objects.nonNull(database.getType()); + } + @Override public void rewrite(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); DatabaseResp database = semanticSchemaResp.getDatabaseResp(); String sql = queryStatement.getSql(); if (Objects.isNull(database) || Objects.isNull(database.getType())) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java deleted file mode 100644 index 92836608a..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/DetailQueryOptimizer.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.optimizer; - -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.StructQueryParam; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import java.util.Objects; - -/** Remove the default metric added by the system when the query only has dimensions */ -@Slf4j -@Component("DetailQueryOptimizer") -public class DetailQueryOptimizer implements QueryOptimizer { - - @Override - public void rewrite(QueryStatement queryStatement) { - StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); - String sqlRaw = queryStatement.getSql().trim(); - if (StringUtils.isEmpty(sqlRaw)) { - throw new RuntimeException("sql is empty or null"); - } - log.debug("before handleNoMetric, sql:{}", sqlRaw); - // if (isDetailQuery(structQueryParam)) { - // if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) { - // String sqlForm = "select %s from ( %s ) src_no_metric"; - // String sql = String.format(sqlForm, - // structQueryParam.getGroups().stream().collect(Collectors.joining(",")), - // sqlRaw); - // queryStatement.setSql(sql); - // } - // } - log.debug("after handleNoMetric, sql:{}", queryStatement.getSql()); - } - - public boolean isDetailQuery(StructQueryParam structQueryParam) { - return Objects.nonNull(structQueryParam) - && structQueryParam.getQueryType().isNativeAggQuery(); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java index baeb626dd..b8ac22bf1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/QueryOptimizer.java @@ -7,5 +7,9 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; * derive the most efficient query. */ public interface QueryOptimizer { + + boolean accept(QueryStatement queryStatement); + void rewrite(QueryStatement queryStatement); + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java index 3d13d2d0a..ed7d56fea 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/optimizer/ResultLimitOptimizer.java @@ -9,10 +9,13 @@ import org.springframework.stereotype.Component; @Component("ResultLimitOptimizer") public class ResultLimitOptimizer implements QueryOptimizer { + @Override + public boolean accept(QueryStatement queryStatement) { + return !SqlSelectHelper.hasLimit(queryStatement.getSql()); + } + @Override public void rewrite(QueryStatement queryStatement) { - if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) { - queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit()); - } + queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit()); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java similarity index 79% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java index 46f3494b5..42c65ab2d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; @@ -22,24 +22,24 @@ import java.util.Objects; import java.util.stream.Collectors; @Slf4j -@Component("DefaultDimValueConverter") -public class DefaultDimValueConverter implements QueryConverter { +@Component("DefaultDimValueParser") +public class DefaultDimValueParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { - return Objects.nonNull(queryStatement.getSqlQueryParam()) - && StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSql()); + return Objects.nonNull(queryStatement.getSqlQuery()) + && StringUtils.isNotBlank(queryStatement.getSqlQuery().getSql()); } @Override - public void convert(QueryStatement queryStatement) { + public void parse(QueryStatement queryStatement) { List dimensions = queryStatement.getOntology().getDimensions().stream() .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(dimensions)) { return; } - String sql = queryStatement.getSqlQueryParam().getSql(); + String sql = queryStatement.getSqlQuery().getSql(); List whereFields = SqlSelectHelper.getWhereFields(sql).stream() .filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) .collect(Collectors.toList()); @@ -56,11 +56,11 @@ public class DefaultDimValueConverter implements QueryConverter { inExpression.setLeftExpression(new Column(dimension.getBizName())); inExpression.setRightExpression(expressionList); expressions.add(inExpression); - if (Objects.nonNull(queryStatement.getSqlQueryParam().getTable())) { - queryStatement.getOntologyQueryParam().getDimensions().add(dimension.getBizName()); + if (Objects.nonNull(queryStatement.getSqlQuery().getTable())) { + queryStatement.getOntologyQuery().getDimensions().add(dimension.getBizName()); } } sql = SqlAddHelper.addWhere(sql, expressions); - queryStatement.getSqlQueryParam().setSql(sql); + queryStatement.getSqlQuery().setSql(sql); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java similarity index 56% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java index 5bc209587..1d81bc42d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MetricRatioParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; @@ -9,9 +9,9 @@ import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; -import com.tencent.supersonic.headless.core.pojo.StructQueryParam; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; +import com.tencent.supersonic.headless.core.pojo.StructQuery; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -22,30 +22,29 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -@Component("CalculateAggConverter") +@Component("MetricRatioParser") @Slf4j -public class MetricRatioConverter implements QueryConverter { +public class MetricRatioParser implements QueryParser { public interface EngineSql { - String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, - String metricSql); + String sql(StructQuery structQuery, boolean isOver, boolean asWith, String metricSql); } @Override public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getStructQueryParam()) || queryStatement.getIsS2SQL() - || !isRatioAccept(queryStatement.getStructQueryParam())) { + if (Objects.isNull(queryStatement.getStructQuery()) || queryStatement.getIsS2SQL() + || !isRatioAccept(queryStatement.getStructQuery())) { return false; } - StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); - if (structQueryParam.getQueryType().isNativeAggQuery() - || CollectionUtils.isEmpty(structQueryParam.getAggregators())) { + StructQuery structQuery = queryStatement.getStructQuery(); + if (structQuery.getQueryType().isNativeAggQuery() + || CollectionUtils.isEmpty(structQuery.getAggregators())) { return false; } int nonSumFunction = 0; - for (Aggregator agg : structQueryParam.getAggregators()) { + for (Aggregator agg : structQuery.getAggregators()) { if (agg.getFunc() == null || "".equals(agg.getFunc())) { return false; } @@ -60,14 +59,14 @@ public class MetricRatioConverter implements QueryConverter { } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { Database database = queryStatement.getOntology().getDatabase(); generateRatioSql(queryStatement, database.getType(), database.getVersion()); } /** Ratio */ - public boolean isRatioAccept(StructQueryParam structQueryParam) { - Long ratioFuncNum = structQueryParam.getAggregators().stream() + public boolean isRatioAccept(StructQuery structQuery) { + Long ratioFuncNum = structQuery.getAggregators().stream() .filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc().equals(AggOperatorEnum.RATIO_OVER))) .count(); @@ -80,20 +79,20 @@ public class MetricRatioConverter implements QueryConverter { public void generateRatioSql(QueryStatement queryStatement, EngineType engineTypeEnum, String version) throws Exception { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); - check(structQueryParam); + StructQuery structQuery = queryStatement.getStructQuery(); + check(structQuery); queryStatement.setEnableOptimize(false); - OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam(); - ontologyQueryParam.setAggOption(AggOption.AGGREGATION); + OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); + ontologyQuery.setAggOption(AggOption.AGGREGATION); String metricTableName = "v_metric_tb_tmp"; - boolean isOver = isOverRatio(structQueryParam); + boolean isOver = isOverRatio(structQuery); String sql = ""; - SqlQueryParam dsParam = queryStatement.getSqlQueryParam(); + SqlQuery dsParam = queryStatement.getSqlQuery(); dsParam.setTable(metricTableName); switch (engineTypeEnum) { case H2: - sql = new H2EngineSql().sql(structQueryParam, isOver, true, metricTableName); + sql = new H2EngineSql().sql(structQuery, isOver, true, metricTableName); break; case MYSQL: case DORIS: @@ -102,10 +101,10 @@ public class MetricRatioConverter implements QueryConverter { dsParam.setSupportWith(false); } if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) { - sql = new MysqlEngineSql().sql(structQueryParam, isOver, - dsParam.isSupportWith(), metricTableName); + sql = new MysqlEngineSql().sql(structQuery, isOver, dsParam.isSupportWith(), + metricTableName); } else { - sql = new CkEngineSql().sql(structQueryParam, isOver, dsParam.isSupportWith(), + sql = new CkEngineSql().sql(structQuery, isOver, dsParam.isSupportWith(), metricTableName); } break; @@ -116,8 +115,8 @@ public class MetricRatioConverter implements QueryConverter { public class H2EngineSql implements EngineSql { - public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) { - String aggStr = structQueryParam.getAggregators().stream().map(f -> { + public String getOverSelect(StructQuery structQuery, boolean isOver) { + String aggStr = structQuery.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", @@ -127,44 +126,43 @@ public class MetricRatioConverter implements QueryConverter { return f.getColumn(); } }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr - : String.join(",", structQueryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQuery.getGroups()) ? aggStr + : String.join(",", structQuery.getGroups()) + "," + aggStr; } - public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver, - boolean isAdd) { - if (Objects.nonNull(structQueryParam.getDateInfo())) { + public String getTimeSpan(StructQuery structQuery, boolean isOver, boolean isAdd) { + if (Objects.nonNull(structQuery.getDateInfo())) { String addStr = isAdd ? "" : "-"; - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { return "day," + (isOver ? addStr + "7" : addStr + "1"); } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return isOver ? "month," + addStr + "1" : "day," + addStr + "7"; } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) { return isOver ? "year," + addStr + "1" : "month," + addStr + "1"; } } return ""; } - public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQuery structQuery, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(structQueryParam); - String timeSpan = getTimeSpan(structQueryParam, isOver, true); - String aggStr = structQueryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQuery); + String timeSpan = getTimeSpan(structQuery, isOver, true); + String aggStr = structQuery.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ", aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim); } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ", - getTimeSpan(structQueryParam, isOver, false), aliasLeft + timeDim, + getTimeSpan(structQuery, isOver, false), aliasLeft + timeDim, aliasRight + timeDim); } return String.format("%s = TIMESTAMPADD(%s,%s) ", aliasLeft + timeDim, timeSpan, @@ -174,7 +172,7 @@ public class MetricRatioConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : structQueryParam.getGroups()) { + for (String group : structQuery.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -185,36 +183,36 @@ public class MetricRatioConverter implements QueryConverter { } @Override - public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + public String sql(StructQuery structQuery, boolean isOver, boolean asWith, String metricSql) { String sql = String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."), - getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, - getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), - getLimit(structQueryParam)); + getOverSelect(structQuery, isOver), getAllSelect(structQuery, "t0."), + getAllJoinSelect(structQuery, "t1."), metricSql, metricSql, + getJoinOn(structQuery, isOver, "t0.", "t1."), getOrderBy(structQuery), + getLimit(structQuery)); return sql; } } public class CkEngineSql extends MysqlEngineSql { - public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQuery structQuery, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(structQueryParam); - String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true); - String aggStr = structQueryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQuery); + String timeSpan = "INTERVAL " + getTimeSpan(structQuery, isOver, true); + String aggStr = structQuery.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ", aliasLeft + timeDim, aliasRight + timeDim, timeSpan); } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false), + aliasLeft + timeDim, getTimeSpan(structQuery, isOver, false), aliasRight + timeDim); } return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim, @@ -224,7 +222,7 @@ public class MetricRatioConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : structQueryParam.getGroups()) { + for (String group : structQuery.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -235,49 +233,46 @@ public class MetricRatioConverter implements QueryConverter { } @Override - public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + public String sql(StructQuery structQuery, boolean isOver, boolean asWith, String metricSql) { if (!asWith) { return String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(structQueryParam, isOver), - getAllSelect(structQueryParam, "t0."), - getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, - getJoinOn(structQueryParam, isOver, "t0.", "t1."), - getOrderBy(structQueryParam), getLimit(structQueryParam)); + getOverSelect(structQuery, isOver), getAllSelect(structQuery, "t0."), + getAllJoinSelect(structQuery, "t1."), metricSql, metricSql, + getJoinOn(structQuery, isOver, "t0.", "t1."), getOrderBy(structQuery), + getLimit(structQuery)); } return String.format( ",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s " + "from t0 left join t1 on %s ) metric_tb_src %s %s ", - metricSql, metricSql, getOverSelect(structQueryParam, isOver), - getAllSelect(structQueryParam, "t0."), - getAllJoinSelect(structQueryParam, "t1."), - getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), - getLimit(structQueryParam)); + metricSql, metricSql, getOverSelect(structQuery, isOver), + getAllSelect(structQuery, "t0."), getAllJoinSelect(structQuery, "t1."), + getJoinOn(structQuery, isOver, "t0.", "t1."), getOrderBy(structQuery), + getLimit(structQuery)); } } public class MysqlEngineSql implements EngineSql { - public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver, - boolean isAdd) { - if (Objects.nonNull(structQueryParam.getDateInfo())) { + public String getTimeSpan(StructQuery structQuery, boolean isOver, boolean isAdd) { + if (Objects.nonNull(structQuery.getDateInfo())) { String addStr = isAdd ? "" : "-"; - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { return isOver ? addStr + "7 day" : addStr + "1 day"; } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) { return isOver ? addStr + "1 month" : addStr + "7 day"; } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return isOver ? addStr + "1 year" : addStr + "1 month"; } } return ""; } - public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) { - String aggStr = structQueryParam.getAggregators().stream().map(f -> { + public String getOverSelect(StructQuery structQuery, boolean isOver) { + String aggStr = structQuery.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return String.format("if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", @@ -287,26 +282,26 @@ public class MetricRatioConverter implements QueryConverter { return f.getColumn(); } }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr - : String.join(",", structQueryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQuery.getGroups()) ? aggStr + : String.join(",", structQuery.getGroups()) + "," + aggStr; } - public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQuery structQuery, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(structQueryParam); - String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true); - String aggStr = structQueryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQuery); + String timeSpan = "INTERVAL " + getTimeSpan(structQuery, isOver, true); + String aggStr = structQuery.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ", aliasLeft + timeDim, aliasRight + timeDim, timeSpan); } - if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQuery.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false), + aliasLeft + timeDim, getTimeSpan(structQuery, isOver, false), aliasRight + timeDim); } return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim, @@ -316,7 +311,7 @@ public class MetricRatioConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : structQueryParam.getGroups()) { + for (String group : structQuery.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -327,53 +322,52 @@ public class MetricRatioConverter implements QueryConverter { } @Override - public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + public String sql(StructQuery structQuery, boolean isOver, boolean asWith, String metricSql) { String sql = String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."), - getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, - getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), - getLimit(structQueryParam)); + getOverSelect(structQuery, isOver), getAllSelect(structQuery, "t0."), + getAllJoinSelect(structQuery, "t1."), metricSql, metricSql, + getJoinOn(structQuery, isOver, "t0.", "t1."), getOrderBy(structQuery), + getLimit(structQuery)); return sql; } } - private String getAllJoinSelect(StructQueryParam structQueryParam, String alias) { - String aggStr = structQueryParam.getAggregators().stream() + private String getAllJoinSelect(StructQuery structQuery, String alias) { + String aggStr = structQuery.getAggregators().stream() .map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") + "_roll") .collect(Collectors.joining(",")); List groups = new ArrayList<>(); - for (String group : structQueryParam.getGroups()) { + for (String group : structQuery.getGroups()) { groups.add(alias + group + " as " + group + "_roll"); } return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr; } - private String getGroupDimWithOutTime(StructQueryParam structQueryParam) { - String timeDim = getTimeDim(structQueryParam); - return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) + private String getGroupDimWithOutTime(StructQuery structQuery) { + String timeDim = getTimeDim(structQuery); + return structQuery.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) .collect(Collectors.joining(",")); } - private static String getTimeDim(StructQueryParam structQueryParam) { + private static String getTimeDim(StructQuery structQuery) { DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class); - return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo()); + return dateModeUtils.getSysDateCol(structQuery.getDateInfo()); } - private static String getLimit(StructQueryParam structQueryParam) { - if (structQueryParam != null && structQueryParam.getLimit() != null - && structQueryParam.getLimit() > 0) { - return " limit " + String.valueOf(structQueryParam.getLimit()); + private static String getLimit(StructQuery structQuery) { + if (structQuery != null && structQuery.getLimit() != null && structQuery.getLimit() > 0) { + return " limit " + String.valueOf(structQuery.getLimit()); } return ""; } - private String getAllSelect(StructQueryParam structQueryParam, String alias) { - String aggStr = structQueryParam.getAggregators().stream() - .map(f -> getSelectField(f, alias)).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr - : alias + String.join("," + alias, structQueryParam.getGroups()) + "," + aggStr; + private String getAllSelect(StructQuery structQuery, String alias) { + String aggStr = structQuery.getAggregators().stream().map(f -> getSelectField(f, alias)) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(structQuery.getGroups()) ? aggStr + : alias + String.join("," + alias, structQuery.getGroups()) + "," + aggStr; } private String getSelectField(final Aggregator agg, String alias) { @@ -385,32 +379,32 @@ public class MetricRatioConverter implements QueryConverter { return sqlGenerateUtils.getSelectField(agg); } - private String getGroupBy(StructQueryParam structQueryParam) { - if (CollectionUtils.isEmpty(structQueryParam.getGroups())) { + private String getGroupBy(StructQuery structQuery) { + if (CollectionUtils.isEmpty(structQuery.getGroups())) { return ""; } - return "group by " + String.join(",", structQueryParam.getGroups()); + return "group by " + String.join(",", structQuery.getGroups()); } - private static String getOrderBy(StructQueryParam structQueryParam) { - return "order by " + getTimeDim(structQueryParam) + " desc"; + private static String getOrderBy(StructQuery structQuery) { + return "order by " + getTimeDim(structQuery) + " desc"; } - private boolean isOverRatio(StructQueryParam structQueryParam) { - Long overCt = structQueryParam.getAggregators().stream() + private boolean isOverRatio(StructQuery structQuery) { + Long overCt = structQuery.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); return overCt > 0; } - private void check(StructQueryParam structQueryParam) throws Exception { - Long ratioOverNum = structQueryParam.getAggregators().stream() + private void check(StructQuery structQuery) throws Exception { + Long ratioOverNum = structQuery.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); - Long ratioRollNum = structQueryParam.getAggregators().stream() + Long ratioRollNum = structQuery.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count(); if (ratioOverNum > 0 && ratioRollNum > 0) { throw new Exception("not support over ratio and roll ratio together "); } - if (getTimeDim(structQueryParam).isEmpty()) { + if (getTimeDim(structQuery).isEmpty()) { throw new Exception("miss time filter"); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java similarity index 57% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java index c4047b6b5..e854e1123 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java @@ -1,24 +1,28 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.parser.QueryParser; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Ontology; +import com.tencent.supersonic.headless.core.translator.parser.calcite.RuntimeOptions; +import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; +import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder; +import com.tencent.supersonic.headless.core.pojo.Ontology; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.Objects; + /** the calcite parse implements */ -@Component("CalciteQueryParser") +@Component("OntologyQueryParser") @Slf4j -public class CalciteQueryParser implements QueryParser { +public class OntologyQueryParser implements QueryParser { + + @Override + public boolean accept(QueryStatement queryStatement) { + return Objects.nonNull(queryStatement.getOntologyQuery()); + } @Override public void parse(QueryStatement queryStatement) throws Exception { Ontology ontology = queryStatement.getOntology(); - if (ontology == null) { - queryStatement.setErrMsg("No ontology could be found"); - return; - } - S2CalciteSchema semanticSchema = S2CalciteSchema.builder() .schemaKey("DATASET_" + queryStatement.getDataSetId()).ontology(ontology) .runtimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java index c3a584832..aa870008c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/QueryParser.java @@ -4,5 +4,8 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** A query parser generates physical SQL for the QueryStatement. */ public interface QueryParser { + + boolean accept(QueryStatement queryStatement); + void parse(QueryStatement queryStatement) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java similarity index 86% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index 9ce495cff..0a4191210 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.common.jsqlparser.SqlAsHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; @@ -14,8 +14,8 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -27,62 +27,62 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -@Component("SqlQueryConverter") +@Component("SqlQueryParser") @Slf4j -public class SqlQueryConverter implements QueryConverter { +public class SqlQueryParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { - return Objects.nonNull(queryStatement.getSqlQueryParam()) && queryStatement.getIsS2SQL(); + return Objects.nonNull(queryStatement.getSqlQuery()) && queryStatement.getIsS2SQL(); } @Override - public void convert(QueryStatement queryStatement) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { convertNameToBizName(queryStatement); rewriteOrderBy(queryStatement); // fill sqlQuery - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam(); - String tableName = SqlSelectHelper.getTableName(sqlQueryParam.getSql()); + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); + SqlQuery sqlQuery = queryStatement.getSqlQuery(); + String tableName = SqlSelectHelper.getTableName(sqlQuery.getSql()); if (StringUtils.isEmpty(tableName)) { return; } - sqlQueryParam.setTable(tableName.toLowerCase()); + sqlQuery.setTable(tableName.toLowerCase()); SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); if (!sqlGenerateUtils.isSupportWith( EngineType.fromString(semanticSchemaResp.getDatabaseResp().getType().toUpperCase()), semanticSchemaResp.getDatabaseResp().getVersion())) { - sqlQueryParam.setSupportWith(false); - sqlQueryParam.setWithAlias(false); + sqlQuery.setSupportWith(false); + sqlQuery.setWithAlias(false); } // build ontologyQuery - List allFields = SqlSelectHelper.getAllSelectFields(sqlQueryParam.getSql()); + List allFields = SqlSelectHelper.getAllSelectFields(sqlQuery.getSql()); List metricSchemas = getMetrics(semanticSchemaResp, allFields); List metrics = metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList()); Set dimensions = getDimensions(semanticSchemaResp, allFields); - OntologyQueryParam ontologyQueryParam = new OntologyQueryParam(); - ontologyQueryParam.getMetrics().addAll(metrics); - ontologyQueryParam.getDimensions().addAll(dimensions); - AggOption sqlQueryAggOption = getAggOption(sqlQueryParam.getSql(), metricSchemas); + OntologyQuery ontologyQuery = new OntologyQuery(); + ontologyQuery.getMetrics().addAll(metrics); + ontologyQuery.getDimensions().addAll(dimensions); + AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), metricSchemas); // if sql query itself has aggregation, ontology query just returns detail if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) { - ontologyQueryParam.setAggOption(AggOption.NATIVE); + ontologyQuery.setAggOption(AggOption.NATIVE); } else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !metrics.isEmpty()) { - ontologyQueryParam.setAggOption(AggOption.DEFAULT); + ontologyQuery.setAggOption(AggOption.DEFAULT); } - ontologyQueryParam.setNativeQuery(!AggOption.isAgg(ontologyQueryParam.getAggOption())); - queryStatement.setOntologyQueryParam(ontologyQueryParam); + ontologyQuery.setNativeQuery(!AggOption.isAgg(ontologyQuery.getAggOption())); + queryStatement.setOntologyQuery(ontologyQuery); generateDerivedMetric(sqlGenerateUtils, queryStatement); - queryStatement.setSql(sqlQueryParam.getSql()); + queryStatement.setSql(sqlQuery.getSql()); // replace sql fields for db, must called after convertNameToBizName - String sqlRewrite = replaceSqlFieldsForHanaDB(queryStatement, sqlQueryParam.getSql()); - sqlQueryParam.setSql(sqlRewrite); - log.info("parse sqlQuery [{}] ", sqlQueryParam); + String sqlRewrite = replaceSqlFieldsForHanaDB(queryStatement, sqlQuery.getSql()); + sqlQuery.setSql(sqlRewrite); + log.info("parse sqlQuery [{}] ", sqlQuery); } private AggOption getAggOption(String sql, List metricSchemas) { @@ -145,9 +145,9 @@ public class SqlQueryConverter implements QueryConverter { private void generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils, QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - SqlQueryParam sqlParam = queryStatement.getSqlQueryParam(); - OntologyQueryParam ontologyParam = queryStatement.getOntologyQueryParam(); + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); + SqlQuery sqlParam = queryStatement.getSqlQuery(); + OntologyQuery ontologyParam = queryStatement.getOntologyQuery(); String sql = sqlParam.getSql(); Set measures = new HashSet<>(); @@ -229,22 +229,20 @@ public class SqlQueryConverter implements QueryConverter { /** - * special process for hanaDB,the sap hana DB don't support the chinese name as - * the column name, - * so we need to quote the column name after converting the convertNameToBizName - * called + * special process for hanaDB,the sap hana DB don't support the chinese name as the column name, + * so we need to quote the column name after converting the convertNameToBizName called * - * sap hana DB will auto translate the colume to upper case letter if not - * quoted. - * also we need to quote the field name if it is a lower case letter. + * sap hana DB will auto translate the colume to upper case letter if not quoted. also we need + * to quote the field name if it is a lower case letter. * * @param queryStatement * @param sql * @return */ private String replaceSqlFieldsForHanaDB(QueryStatement queryStatement, String sql) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - if (!semanticSchemaResp.getDatabaseResp().getType().equalsIgnoreCase(EngineType.HANADB.getName())) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); + if (!semanticSchemaResp.getDatabaseResp().getType() + .equalsIgnoreCase(EngineType.HANADB.getName())) { return sql; } Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); @@ -277,9 +275,9 @@ public class SqlQueryConverter implements QueryConverter { } private void convertNameToBizName(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); - String sql = queryStatement.getSqlQueryParam().getSql(); + String sql = queryStatement.getSqlQuery().getSql(); log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(), sql); sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); @@ -288,15 +286,15 @@ public class SqlQueryConverter implements QueryConverter { sql = SqlReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + queryStatement.getDataSetId()); log.debug("replaceTableName after:{}", sql); - queryStatement.getSqlQueryParam().setSql(sql); + queryStatement.getSqlQuery().setSql(sql); } private void rewriteOrderBy(QueryStatement queryStatement) { // replace order by field with the select sequence number - String sql = queryStatement.getSqlQueryParam().getSql(); + String sql = queryStatement.getSqlQuery().getSql(); String newSql = SqlReplaceHelper.replaceAggAliasOrderbyField(sql); log.debug("replaceOrderAggSameAlias {} -> {}", sql, newSql); - queryStatement.getSqlQueryParam().setSql(newSql); + queryStatement.getSqlQuery().setSql(newSql); } protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java similarity index 77% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java index 1ef5c2147..70db3f33c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlVariableParser.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.converter; +package com.tencent.supersonic.headless.core.translator.parser; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; @@ -14,18 +14,17 @@ import java.util.List; import java.util.Objects; @Slf4j -@Component("SqlVariableConverter") -public class SqlVariableConverter implements QueryConverter { +@Component("SqlVariableParser") +public class SqlVariableParser implements QueryParser { @Override public boolean accept(QueryStatement queryStatement) { - return Objects.nonNull(queryStatement.getStructQueryParam()) - && !queryStatement.getIsS2SQL(); + return Objects.nonNull(queryStatement.getStructQuery()) && !queryStatement.getIsS2SQL(); } @Override - public void convert(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + public void parse(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); List modelResps = semanticSchemaResp.getModelResps(); if (CollectionUtils.isEmpty(modelResps)) { return; @@ -36,7 +35,7 @@ public class SqlVariableConverter implements QueryConverter { String sqlParsed = SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), modelResp.getModelDetail().getSqlVariables(), - queryStatement.getStructQueryParam().getParams()); + queryStatement.getStructQuery().getParams()); DataModel dataModel = queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName()); dataModel.setSqlQuery(sqlParsed); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java new file mode 100644 index 000000000..8e38780ed --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/StructQueryParser.java @@ -0,0 +1,72 @@ +package com.tencent.supersonic.headless.core.translator.parser; + +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.ColumnOrder; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; +import com.tencent.supersonic.headless.core.pojo.StructQuery; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.Objects; +import java.util.stream.Collectors; + +@Component("StructQueryParser") +@Slf4j +public class StructQueryParser implements QueryParser { + + @Override + public boolean accept(QueryStatement queryStatement) { + return Objects.nonNull(queryStatement.getStructQuery()) && !queryStatement.getIsS2SQL(); + } + + @Override + public void parse(QueryStatement queryStatement) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); + StructQuery structQuery = queryStatement.getStructQuery(); + + String dsTable = "t_1"; + SqlQuery sqlParam = new SqlQuery(); + sqlParam.setTable(dsTable); + String sql = String.format("select %s from %s %s %s %s", + sqlGenerateUtils.getSelect(structQuery), dsTable, + sqlGenerateUtils.getGroupBy(structQuery), sqlGenerateUtils.getOrderBy(structQuery), + sqlGenerateUtils.getLimit(structQuery)); + Database database = queryStatement.getOntology().getDatabase(); + if (!sqlGenerateUtils.isSupportWith(database.getType(), database.getVersion())) { + sqlParam.setSupportWith(false); + sql = String.format("select %s from %s t0 %s %s %s", + sqlGenerateUtils.getSelect(structQuery), dsTable, + sqlGenerateUtils.getGroupBy(structQuery), + sqlGenerateUtils.getOrderBy(structQuery), + sqlGenerateUtils.getLimit(structQuery)); + } + sqlParam.setSql(sql); + queryStatement.setSqlQuery(sqlParam); + + OntologyQuery ontologyQuery = new OntologyQuery(); + ontologyQuery.getDimensions().addAll(structQuery.getGroups()); + ontologyQuery.getMetrics().addAll(structQuery.getAggregators().stream() + .map(Aggregator::getColumn).collect(Collectors.toList())); + String where = sqlGenerateUtils.generateWhere(structQuery, null); + ontologyQuery.setWhere(where); + if (ontologyQuery.getMetrics().isEmpty()) { + ontologyQuery.setAggOption(AggOption.NATIVE); + } else { + ontologyQuery.setAggOption(AggOption.DEFAULT); + } + ontologyQuery.setNativeQuery(structQuery.getQueryType().isNativeAggQuery()); + ontologyQuery.setOrder(structQuery.getOrders().stream() + .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) + .collect(Collectors.toList())); + ontologyQuery.setLimit(structQuery.getLimit()); + queryStatement.setOntologyQuery(ontologyQuery); + log.info("parse structQuery [{}] ", queryStatement.getSqlQuery()); + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java index e0e0af42c..b35dd6b9d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Ontology; +import com.tencent.supersonic.headless.core.pojo.Ontology; import lombok.Builder; import lombok.Data; import org.apache.calcite.schema.Schema; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java index c8e48b352..c46da6fe6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java @@ -13,7 +13,7 @@ import com.tencent.supersonic.headless.core.translator.parser.calcite.render.Ren import com.tencent.supersonic.headless.core.translator.parser.calcite.render.SourceRender; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; @@ -25,7 +25,7 @@ import java.util.*; public class SqlBuilder { private final S2CalciteSchema schema; - private OntologyQueryParam ontologyQueryParam; + private OntologyQuery ontologyQuery; private SqlValidatorScope scope; private SqlNode parserNode; private boolean isAgg = false; @@ -36,11 +36,11 @@ public class SqlBuilder { } public String buildOntologySql(QueryStatement queryStatement) throws Exception { - this.ontologyQueryParam = queryStatement.getOntologyQueryParam(); - if (ontologyQueryParam.getLimit() == null) { - ontologyQueryParam.setLimit(0L); + this.ontologyQuery = queryStatement.getOntologyQuery(); + if (ontologyQuery.getLimit() == null) { + ontologyQuery.setLimit(0L); } - this.aggOption = ontologyQueryParam.getAggOption(); + this.aggOption = ontologyQuery.getAggOption(); buildParseNode(); Database database = queryStatement.getOntology().getDatabase(); @@ -51,8 +51,7 @@ public class SqlBuilder { private void buildParseNode() throws Exception { // find relevant data models scope = SchemaBuilder.getScope(schema); - List dataModels = - DataModelNode.getQueryDataModels(scope, schema, ontologyQueryParam); + List dataModels = DataModelNode.getQueryDataModels(scope, schema, ontologyQuery); if (dataModels == null || dataModels.isEmpty()) { throw new Exception("data model not found"); } @@ -69,14 +68,14 @@ public class SqlBuilder { while (it.hasNext()) { Renderer renderer = it.next(); if (previous != null) { - previous.render(ontologyQueryParam, dataModels, scope, schema, !isAgg); + previous.render(ontologyQuery, dataModels, scope, schema, !isAgg); renderer.setTable(previous .builderAs(DataModelNode.getNames(dataModels) + "_" + String.valueOf(i))); i++; } previous = renderer; } - builders.getLast().render(ontologyQueryParam, dataModels, scope, schema, !isAgg); + builders.getLast().render(ontologyQuery, dataModels, scope, schema, !isAgg); parserNode = builders.getLast().builder(); } @@ -87,7 +86,7 @@ public class SqlBuilder { // default by dataModel time aggregation if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime() .equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { - if (!ontologyQueryParam.isNativeQuery()) { + if (!ontologyQuery.isNativeQuery()) { return true; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java index 95ea823b6..52ca7262c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java @@ -4,6 +4,9 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.pojo.JoinRelation; +import com.tencent.supersonic.headless.core.pojo.Ontology; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.parser.calcite.SchemaBuilder; import com.tencent.supersonic.headless.core.translator.parser.s2sql.*; @@ -130,8 +133,8 @@ public class DataModelNode extends SemanticNode { return dataModelList.stream().map(DataModel::getName).collect(Collectors.joining("_")); } - public static void getQueryDimensionMeasure(Ontology ontology, OntologyQueryParam queryParam, - Set queryDimensions, Set queryMeasures) { + public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery queryParam, + Set queryDimensions, Set queryMeasures) { queryDimensions.addAll(queryParam.getDimensions().stream() .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] @@ -146,9 +149,9 @@ public class DataModelNode extends SemanticNode { .forEach(queryMeasures::add); } - public static void mergeQueryFilterDimensionMeasure(Ontology ontology, - OntologyQueryParam queryParam, Set dimensions, Set measures, - SqlValidatorScope scope) throws Exception { + public static void mergeQueryFilterDimensionMeasure(Ontology ontology, OntologyQuery queryParam, + Set dimensions, Set measures, SqlValidatorScope scope) + throws Exception { EngineType engineType = ontology.getDatabase().getType(); if (Objects.nonNull(queryParam.getWhere()) && !queryParam.getWhere().isEmpty()) { Set filterConditions = new HashSet<>(); @@ -173,7 +176,7 @@ public class DataModelNode extends SemanticNode { } public static List getQueryDataModels(SqlValidatorScope scope, - S2CalciteSchema schema, OntologyQueryParam queryParam) throws Exception { + S2CalciteSchema schema, OntologyQuery queryParam) throws Exception { Ontology ontology = schema.getOntology(); // get query measures and dimensions Set queryMeasures = new HashSet<>(); @@ -282,7 +285,7 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByRelation(Ontology ontology, - OntologyQueryParam queryParam, DataModel baseDataModel, Set queryDimensions, + OntologyQuery queryParam, DataModel baseDataModel, Set queryDimensions, Set queryMeasures) { Set joinDataModelNames = new HashSet<>(); List joinDataModels = new ArrayList<>(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java index 847aacb68..4f38695a1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java @@ -86,7 +86,7 @@ public abstract class SemanticNode { expression = String.format("`%s`", expression); } } - SqlParser sqlParser = + SqlParser sqlParser = SqlParser.create(expression, Configuration.getParserConfig(engineType)); SqlNode sqlNode = sqlParser.parseExpression(); scope.validateExpr(sqlNode); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/FilterRender.java index 58993f49c..7236bbf7f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/FilterRender.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.translator.parser.calcite.node.Seman import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParserPos; @@ -26,7 +26,7 @@ import java.util.stream.Collectors; public class FilterRender extends Renderer { @Override - public void render(OntologyQueryParam metricCommand, List dataModels, + public void render(OntologyQuery metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView tableView = super.tableView; SqlNode filterNode = null; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java index 3ae790ce9..7f81b647b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java @@ -13,10 +13,10 @@ import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Identify; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.SqlBasicCall; @@ -47,7 +47,7 @@ import java.util.stream.Collectors; public class JoinRender extends Renderer { @Override - public void render(OntologyQueryParam metricCommand, List dataModels, + public void render(OntologyQuery metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); EngineType engineType = schema.getOntology().getDatabase().getType(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/OutputRender.java index abee245bd..a0c1f5d37 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/OutputRender.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.core.translator.parser.calcite.TableView; import com.tencent.supersonic.headless.core.translator.parser.calcite.node.MetricNode; import com.tencent.supersonic.headless.core.translator.parser.calcite.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.fun.SqlStdOperatorTable; @@ -22,7 +22,7 @@ import java.util.List; public class OutputRender extends Renderer { @Override - public void render(OntologyQueryParam metricCommand, List dataModels, + public void render(OntologyQuery metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView selectDataSet = super.tableView; EngineType engineType = schema.getOntology().getDatabase().getType(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java index dfcac8916..94917400a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import lombok.Data; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -114,6 +114,6 @@ public abstract class Renderer { return SemanticNode.buildAs(alias, tableView.build()); } - public abstract void render(OntologyQueryParam metricCommand, List dataModels, + public abstract void render(OntologyQuery metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/SourceRender.java index 8ac3e0678..e6ad9ead6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/SourceRender.java @@ -16,7 +16,7 @@ import com.tencent.supersonic.headless.core.translator.parser.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -334,9 +334,9 @@ public class SourceRender extends Renderer { } } - public void render(OntologyQueryParam ontologyQueryParam, List dataModels, + public void render(OntologyQuery ontologyQuery, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { - String queryWhere = ontologyQueryParam.getWhere(); + String queryWhere = ontologyQuery.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); EngineType engineType = schema.getOntology().getDatabase().getType(); @@ -347,13 +347,13 @@ public class SourceRender extends Renderer { } if (dataModels.size() == 1) { DataModel dataModel = dataModels.get(0); - super.tableView = renderOne("", fieldWhere, ontologyQueryParam.getMetrics(), - ontologyQueryParam.getDimensions(), ontologyQueryParam.getWhere(), dataModel, - scope, schema, nonAgg); + super.tableView = renderOne("", fieldWhere, ontologyQuery.getMetrics(), + ontologyQuery.getDimensions(), ontologyQuery.getWhere(), dataModel, scope, + schema, nonAgg); return; } JoinRender joinRender = new JoinRender(); - joinRender.render(ontologyQueryParam, dataModels, scope, schema, nonAgg); + joinRender.render(ontologyQuery, dataModels, scope, schema, nonAgg); super.tableView = joinRender.getTableView(); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index b6e573e6d..edce8fdc3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryAccelerator; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import lombok.extern.slf4j.Slf4j; @@ -20,29 +19,20 @@ import java.util.stream.Collectors; @Slf4j public class ComponentFactory { - private static List queryConverters = new ArrayList<>(); private static Map queryOptimizers = new HashMap<>(); private static List queryExecutors = new ArrayList<>(); private static List queryAccelerators = new ArrayList<>(); - private static QueryParser queryParser; + private static List queryParsers = new ArrayList<>(); private static QueryCache queryCache; static { - initQueryConverter(); initQueryOptimizer(); initQueryExecutors(); initQueryAccelerators(); - initQueryParser(); + initQueryParsers(); initQueryCache(); } - public static List getQueryConverters() { - if (queryConverters.isEmpty()) { - initQueryConverter(); - } - return queryConverters; - } - public static List getQueryOptimizers() { if (queryOptimizers.isEmpty()) { initQueryOptimizer(); @@ -64,11 +54,11 @@ public class ComponentFactory { return queryAccelerators; } - public static QueryParser getQueryParser() { - if (queryParser == null) { - initQueryParser(); + public static List getQueryParser() { + if (queryParsers.isEmpty()) { + initQueryParsers(); } - return queryParser; + return queryParsers; } public static QueryCache getQueryCache() { @@ -92,23 +82,15 @@ public class ComponentFactory { } private static void initQueryExecutors() { - // queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", - // JdbcExecutor.class)); init(QueryExecutor.class, queryExecutors); } private static void initQueryAccelerators() { - // queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", - // JdbcExecutor.class)); init(QueryAccelerator.class, queryAccelerators); } - private static void initQueryConverter() { - init(QueryConverter.class, queryConverters); - } - - private static void initQueryParser() { - queryParser = init(QueryParser.class); + private static void initQueryParsers() { + init(QueryParser.class, queryParsers); } private static void initQueryCache() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 8adf84d3e..ba7bd31e4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -19,7 +19,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.core.config.ExecutorConfig; -import com.tencent.supersonic.headless.core.pojo.StructQueryParam; +import com.tencent.supersonic.headless.core.pojo.StructQuery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -85,26 +85,25 @@ public class SqlGenerateUtils { return selectSql; } - public String getLimit(StructQueryParam structQueryParam) { - if (structQueryParam != null && structQueryParam.getLimit() != null - && structQueryParam.getLimit() > 0) { - return " limit " + structQueryParam.getLimit(); + public String getLimit(StructQuery structQuery) { + if (structQuery != null && structQuery.getLimit() != null && structQuery.getLimit() > 0) { + return " limit " + structQuery.getLimit(); } return ""; } - public String getSelect(StructQueryParam structQueryParam) { - String aggStr = structQueryParam.getAggregators().stream().map(this::getSelectField) + public String getSelect(StructQuery structQuery) { + String aggStr = structQuery.getAggregators().stream().map(this::getSelectField) .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr - : String.join(",", structQueryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQuery.getGroups()) ? aggStr + : String.join(",", structQuery.getGroups()) + "," + aggStr; } - public String getSelect(StructQueryParam structQueryParam, Map deriveMetrics) { - String aggStr = structQueryParam.getAggregators().stream() + public String getSelect(StructQuery structQuery, Map deriveMetrics) { + String aggStr = structQuery.getAggregators().stream() .map(a -> getSelectField(a, deriveMetrics)).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr - : String.join(",", structQueryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQuery.getGroups()) ? aggStr + : String.join(",", structQuery.getGroups()) + "," + aggStr; } public String getSelectField(final Aggregator agg) { @@ -129,46 +128,46 @@ public class SqlGenerateUtils { return deriveMetrics.get(agg.getColumn()); } - public String getGroupBy(StructQueryParam structQueryParam) { - if (CollectionUtils.isEmpty(structQueryParam.getGroups())) { + public String getGroupBy(StructQuery structQuery) { + if (CollectionUtils.isEmpty(structQuery.getGroups())) { return ""; } - return "group by " + String.join(",", structQueryParam.getGroups()); + return "group by " + String.join(",", structQuery.getGroups()); } - public String getOrderBy(StructQueryParam structQueryParam) { - if (CollectionUtils.isEmpty(structQueryParam.getOrders())) { + public String getOrderBy(StructQuery structQuery) { + if (CollectionUtils.isEmpty(structQuery.getOrders())) { return ""; } - return "order by " + structQueryParam.getOrders().stream() + return "order by " + structQuery.getOrders().stream() .map(order -> " " + order.getColumn() + " " + order.getDirection() + " ") .collect(Collectors.joining(",")); } - public String getOrderBy(StructQueryParam structQueryParam, Map deriveMetrics) { - if (CollectionUtils.isEmpty(structQueryParam.getOrders())) { + public String getOrderBy(StructQuery structQuery, Map deriveMetrics) { + if (CollectionUtils.isEmpty(structQuery.getOrders())) { return ""; } - if (!structQueryParam.getOrders().stream() + if (!structQuery.getOrders().stream() .anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { - return getOrderBy(structQueryParam); + return getOrderBy(structQuery); } - return "order by " + structQueryParam.getOrders().stream() + return "order by " + structQuery.getOrders().stream() .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn()) : order.getColumn()) + " " + order.getDirection() + " ") .collect(Collectors.joining(",")); } - public String generateWhere(StructQueryParam structQueryParam, ItemDateResp itemDateResp) { + public String generateWhere(StructQuery structQuery, ItemDateResp itemDateResp) { String whereClauseFromFilter = - sqlFilterUtils.getWhereClause(structQueryParam.getDimensionFilters()); - String whereFromDate = getDateWhereClause(structQueryParam.getDateInfo(), itemDateResp); - return mergeDateWhereClause(structQueryParam, whereClauseFromFilter, whereFromDate); + sqlFilterUtils.getWhereClause(structQuery.getDimensionFilters()); + String whereFromDate = getDateWhereClause(structQuery.getDateInfo(), itemDateResp); + return mergeDateWhereClause(structQuery, whereClauseFromFilter, whereFromDate); } - private String mergeDateWhereClause(StructQueryParam structQueryParam, - String whereClauseFromFilter, String whereFromDate) { + private String mergeDateWhereClause(StructQuery structQuery, String whereClauseFromFilter, + String whereFromDate) { if (StringUtils.isNotEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) { return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); @@ -180,7 +179,7 @@ public class SqlGenerateUtils { return whereFromDate; } else if (Objects.isNull(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) { log.debug("the current date information is empty, enter the date initialization logic"); - return dateModeUtils.defaultRecentDateInfo(structQueryParam.getDateInfo()); + return dateModeUtils.defaultRecentDateInfo(structQuery.getDateInfo()); } return whereClauseFromFilter; } @@ -204,12 +203,12 @@ public class SqlGenerateUtils { return dateModeUtils.getDateWhereStr(dateInfo, dateDate); } - public Triple getBeginEndTime(StructQueryParam structQueryParam, + public Triple getBeginEndTime(StructQuery structQuery, ItemDateResp dataDate) { - if (Objects.isNull(structQueryParam.getDateInfo())) { + if (Objects.isNull(structQuery.getDateInfo())) { return Triple.of("", "", ""); } - DateConf dateConf = structQueryParam.getDateInfo(); + DateConf dateConf = structQuery.getDateInfo(); String dateInfo = dateModeUtils.getSysDateCol(dateConf); if (dateInfo.isEmpty()) { return Triple.of("", "", ""); diff --git a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java index c85fe7141..620b716e5 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.parser.aggregate; import com.alibaba.fastjson.JSON; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser; +import com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser; import org.junit.jupiter.api.Test; import org.testng.Assert; @@ -316,7 +316,7 @@ public class CalciteSqlParserTest { + " \"createdAt\": 1711367511146,\n" + " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}"; QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class); - CalciteQueryParser calciteSqlParser = new CalciteQueryParser(); + OntologyQueryParser calciteSqlParser = new OntologyQueryParser(); calciteSqlParser.parse(queryStatement); Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""), "SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`" diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index 22d42c9c9..f2cb5b780 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -19,8 +19,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; -import com.tencent.supersonic.headless.core.pojo.StructQueryParam; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; +import com.tencent.supersonic.headless.core.pojo.StructQuery; import com.tencent.supersonic.headless.core.translator.SemanticTranslator; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.server.annotation.S2DataPermission; @@ -128,8 +128,7 @@ public class S2SemanticLayerService implements SemanticLayerService { for (QueryExecutor queryExecutor : queryExecutors) { if (queryExecutor.accept(queryStatement)) { queryResp = queryExecutor.execute(queryStatement); - queryUtils.populateQueryColumns(queryResp, - queryStatement.getSemanticSchemaResp()); + queryUtils.populateQueryColumns(queryResp, queryStatement.getSemanticSchema()); } } @@ -299,7 +298,7 @@ public class S2SemanticLayerService implements SemanticLayerService { QueryStatement queryStatement = new QueryStatement(); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setDataSetId(queryReq.getDataSetId()); - queryStatement.setSemanticSchemaResp(semanticSchemaResp); + queryStatement.setSemanticSchema(semanticSchemaResp); queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp)); return queryStatement; } @@ -308,9 +307,9 @@ public class S2SemanticLayerService implements SemanticLayerService { QueryStatement queryStatement = buildQueryStatement(querySqlReq); queryStatement.setIsS2SQL(true); - SqlQueryParam sqlQueryParam = new SqlQueryParam(); - sqlQueryParam.setSql(querySqlReq.getSql()); - queryStatement.setSqlQueryParam(sqlQueryParam); + SqlQuery sqlQuery = new SqlQuery(); + sqlQuery.setSql(querySqlReq.getSql()); + queryStatement.setSqlQuery(sqlQuery); // If dataSetId or DataSetName is empty, parse dataSetId from the SQL if (querySqlReq.needGetDataSetId()) { @@ -322,9 +321,9 @@ public class S2SemanticLayerService implements SemanticLayerService { private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) { QueryStatement queryStatement = buildQueryStatement(queryReq); - StructQueryParam structQueryParam = new StructQueryParam(); - BeanUtils.copyProperties(queryReq, structQueryParam); - queryStatement.setStructQueryParam(structQueryParam); + StructQuery structQuery = new StructQuery(); + BeanUtils.copyProperties(queryReq, structQuery); + queryStatement.setStructQuery(structQuery); queryStatement.setIsS2SQL(false); return queryStatement; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index d1b99f0b2..9cac5dff1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -4,6 +4,8 @@ import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.pojo.JoinRelation; +import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.parser.s2sql.*; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType; @@ -215,7 +217,7 @@ public class SemanticSchemaManager { } private static List getJoinRelation(List modelRelas, - Map modelIdName) { + Map modelIdName) { List joinRelations = new ArrayList<>(); modelRelas.stream().forEach(r -> { if (modelIdName.containsKey(r.getFromModelId()) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java index 5c6de4df9..8489df62a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java @@ -31,7 +31,7 @@ public class MetricDrillDownChecker { private MetricService metricService; public void checkQuery(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); String sql = queryStatement.getSql(); if (StringUtils.isBlank(sql)) { return; diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index dc69e6d70..169feabd9 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.yaml.*; import lombok.extern.slf4j.Slf4j; @@ -19,8 +19,8 @@ import java.util.List; @Slf4j class HeadlessParserServiceTest { - public static SqlParserResp parser(S2CalciteSchema semanticSchema, - OntologyQueryParam ontologyQueryParam, boolean isAgg) { + public static SqlParserResp parser(S2CalciteSchema semanticSchema, OntologyQuery ontologyQuery, + boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); try { if (semanticSchema == null) { @@ -29,14 +29,14 @@ class HeadlessParserServiceTest { } SqlBuilder aggBuilder = new SqlBuilder(semanticSchema); QueryStatement queryStatement = new QueryStatement(); - queryStatement.setOntologyQueryParam(ontologyQueryParam); + queryStatement.setOntologyQuery(ontologyQuery); String sql = aggBuilder.buildOntologySql(queryStatement); queryStatement.setSql(sql); EngineType engineType = semanticSchema.getOntology().getDatabase().getType(); sqlParser.setSql(aggBuilder.getSql(engineType)); } catch (Exception e) { sqlParser.setErrMsg(e.getMessage()); - log.error("parser error metricQueryReq[{}] error [{}]", ontologyQueryParam, e); + log.error("parser error metricQueryReq[{}] error [{}]", ontologyQuery, e); } return sqlParser; } @@ -155,7 +155,7 @@ class HeadlessParserServiceTest { // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); - OntologyQueryParam metricCommand = new OntologyQueryParam(); + OntologyQuery metricCommand = new OntologyQuery(); metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date"))); metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv"))); metricCommand.setWhere( @@ -168,7 +168,7 @@ class HeadlessParserServiceTest { addDepartment(semanticSchema); - OntologyQueryParam metricCommand2 = new OntologyQueryParam(); + OntologyQuery metricCommand2 = new OntologyQuery(); metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv"))); diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index fee393d9f..6cc3ccf82 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -25,21 +25,18 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ ### headless-core SPIs -com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ - com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter +com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ + com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\ + com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ - com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer -com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ - com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser - com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.JdbcExecutor diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index d90db9e39..fcd7f0c57 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -25,21 +25,18 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ ### headless-core SPIs -com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ - com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\ - com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter +com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ + com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\ + com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\ + com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\ + com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ - com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer -com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ - com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser - com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.JdbcExecutor