From c14d4e59d4af60ffd9bbfee02576392113120db1 Mon Sep 17 00:00:00 2001 From: jipengli Date: Wed, 11 Oct 2023 14:44:54 +0800 Subject: [PATCH] (improvement)(semantic) metric table add agg option --- .../semantic/api/query/enums/AggOption.java | 15 +++++++++++ .../semantic/api/query/pojo/MetricTable.java | 3 ++- .../semantic/query/parser/QueryParser.java | 7 +++--- .../semantic/query/parser/SqlParser.java | 3 ++- .../parser/calcite/CalciteSqlParser.java | 3 ++- .../parser/calcite/planner/AggPlanner.java | 25 +++++++++++++------ .../query/parser/calcite/planner/Planner.java | 3 ++- .../parser/convert/CalculateAggConverter.java | 5 ++-- .../calcite/SemanticParserServiceTest.java | 3 ++- 9 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/enums/AggOption.java diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/enums/AggOption.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/enums/AggOption.java new file mode 100644 index 000000000..14c07b2ef --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/enums/AggOption.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.semantic.api.query.enums; + +public enum AggOption { + NATIVE, + AGGREGATION, + DEFAULT; + + public static AggOption getAggregation(boolean isNativeQuery) { + return isNativeQuery ? NATIVE : AGGREGATION; + } + + public static boolean isAgg(AggOption aggOption) { + return NATIVE.equals(aggOption) ? false : true; + } +} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/MetricTable.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/MetricTable.java index 3a3fa9a57..4b8e549b8 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/MetricTable.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/MetricTable.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.api.query.pojo; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import java.util.List; import lombok.Data; @@ -10,6 +11,6 @@ public class MetricTable { private List metrics; private List dimensions; private String where; - private boolean isAgg = false; + private AggOption aggOption = AggOption.DEFAULT; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java index 3124b1416..03ac0760d 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.query.parser; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; @@ -60,7 +61,7 @@ public class QueryParser { metricReq.setDimensions(metricTable.getDimensions()); metricReq.setWhere(formatWhere(metricTable.getWhere())); metricReq.setRootPath(sqlCommend.getRootPath()); - QueryStatement tableSql = parser(metricReq, metricTable.isAgg()); + QueryStatement tableSql = parser(metricReq, metricTable.getAggOption()); if (!tableSql.isOk()) { queryStatement.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(), tableSql.getErrMsg())); @@ -96,10 +97,10 @@ public class QueryParser { } public QueryStatement parser(MetricReq metricCommand) { - return parser(metricCommand, !metricCommand.isNativeQuery()); + return parser(metricCommand, AggOption.getAggregation(metricCommand.isNativeQuery())); } - public QueryStatement parser(MetricReq metricCommand, boolean isAgg) { + public QueryStatement parser(MetricReq metricCommand, AggOption isAgg) { log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg); QueryStatement queryStatement = new QueryStatement(); if (metricCommand.getRootPath().isEmpty()) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java index c68ec43ce..df6d6b99e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java @@ -1,9 +1,10 @@ package com.tencent.supersonic.semantic.query.parser; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; public interface SqlParser { - QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception; + QueryStatement explain(MetricReq metricReq, AggOption aggOption, Catalog catalog) throws Exception; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java index a1b60c3c7..96a71f4e3 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.parser.SqlParser; @@ -19,7 +20,7 @@ public class CalciteSqlParser implements SqlParser { } @Override - public QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception { + public QueryStatement explain(MetricReq metricReq, AggOption isAgg, Catalog catalog) throws Exception { QueryStatement queryStatement = new QueryStatement(); SemanticModel semanticModel = semanticSchemaManager.get(metricReq.getRootPath()); if (semanticModel == null) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java index 8e6c35958..b2f52fa8e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; @@ -30,7 +31,8 @@ public class AggPlanner implements Planner { private Stack dataSets = new Stack<>(); private SqlNode parserNode; private String sourceId; - private boolean isAgg = true; + private boolean isAgg = false; + private AggOption aggOption = AggOption.DEFAULT; public AggPlanner(SemanticSchema schema) { this.schema = schema; @@ -44,10 +46,7 @@ public class AggPlanner implements Planner { if (datasource == null || datasource.isEmpty()) { throw new Exception("datasource not found"); } - if (Objects.nonNull(datasource.get(0).getAggTime()) && !datasource.get(0).getAggTime().equalsIgnoreCase( - Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { - isAgg = true; - } + isAgg = getAgg(datasource.get(0)); sourceId = String.valueOf(datasource.get(0).getSourceId()); // build level by level @@ -78,9 +77,21 @@ public class AggPlanner implements Planner { return DataSourceNode.getMatchDataSources(scope, schema, metricCommand); } + private boolean getAgg(DataSource dataSource) { + if (!AggOption.DEFAULT.equals(aggOption)) { + return AggOption.isAgg(aggOption); + } + // default by dataSource time aggregation + if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime().equalsIgnoreCase( + Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { + return true; + } + return isAgg; + } + @Override - public void explain(MetricReq metricCommand, boolean isAgg) throws Exception { + public void explain(MetricReq metricCommand, AggOption aggOption) throws Exception { this.metricCommand = metricCommand; if (metricCommand.getMetrics() == null) { metricCommand.setMetrics(new ArrayList<>()); @@ -91,7 +102,7 @@ public class AggPlanner implements Planner { if (metricCommand.getLimit() == null) { metricCommand.setLimit(0L); } - this.isAgg = isAgg; + this.aggOption = aggOption; // build a parse Node parse(); // optimizer diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/Planner.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/Planner.java index 353e7bcda..5df59e7a8 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/Planner.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/Planner.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; public interface Planner { - public void explain(MetricReq metricCommand, boolean isAgg) throws Exception; + public void explain(MetricReq metricCommand, AggOption aggOption) throws Exception; public String getSql(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java index b092e0511..c0f3ba01a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; @@ -75,7 +76,7 @@ public class CalculateAggConverter implements SemanticConverter { String where = queryStructUtils.generateWhere(queryStructCmd); log.info("in generateSqlCommand, complete where:{}", where); metricTable.setWhere(where); - metricTable.setAgg(true); + metricTable.setAggOption(AggOption.AGGREGATION); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd), metricTableName, @@ -159,7 +160,7 @@ public class CalculateAggConverter implements SemanticConverter { String where = queryStructUtils.generateWhere(queryStructCmd); log.info("in generateSqlCommend, complete where:{}", where); metricTable.setWhere(where); - metricTable.setAgg(true); + metricTable.setAggOption(AggOption.AGGREGATION); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); boolean isOver = isOverRatio(queryStructCmd); String sql = ""; diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java index de7090c61..42b24f6a1 100644 --- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.api.model.response.SqlParserResp; +import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager; @@ -37,7 +38,7 @@ class SemanticParserServiceTest { return sqlParser; } AggPlanner aggBuilder = new AggPlanner(semanticSchema); - aggBuilder.explain(metricCommand, isAgg); + aggBuilder.explain(metricCommand, AggOption.getAggregation(!isAgg)); sqlParser.setSql(aggBuilder.getSql()); sqlParser.setSourceId(aggBuilder.getSourceId()); } catch (Exception e) {