topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) code refactor (#724)

Browse Source
This commit is contained in:
jipeli
2024-02-07 14:28:01 +08:00
committed by GitHub
parent 9600456bae
commit 329ad327b0
22 changed files with 438 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java Normal file
View File

@@ -0,0 +1,42 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.Data;
@Data
public class QueryParam {
// struct
private List<String> groups = new ArrayList();
private List<Aggregator> aggregators = new ArrayList();
private List<Order> orders = new ArrayList();
private List<Filter> dimensionFilters = new ArrayList();
private List<Filter> metricFilters = new ArrayList();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType;
private String s2SQL;
private String correctS2SQL;
private Long viewId;
private String viewName;
private Set<Long> modelIds = new HashSet<>();
private List<Param> params = new ArrayList<>();
// metric
private List<String> metrics = new ArrayList();
private List<String> dimensions;
private Map<String, String> variables;
private String where;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

88
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java
View File

@@ -2,52 +2,51 @@ package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component @Component
@Slf4j @Slf4j
@Primary @Primary
public class DefaultQueryParser implements QueryParser { public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getParseSqlReq())) { if (Objects.isNull(queryStatement.getViewQueryParam())) {
queryStatement.setParseSqlReq(new ParseSqlReq()); queryStatement.setViewQueryParam(new ViewQueryParam());
} }
if (Objects.isNull(queryStatement.getMetricReq())) { if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricReq(new MetricQueryReq()); queryStatement.setMetricQueryParam(new MetricQueryParam());
} }
log.info("SemanticConverter before [{}]", queryStructReq); log.info("SemanticConverter before [{}]", queryParam);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) { for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) { if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName()); log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement); headlessConverter.convert(queryStatement);
} }
} }
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(), log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getViewQueryParam(),
queryStatement.getMetricReq()); queryStatement.getMetricQueryParam());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) { if (!queryStatement.getViewQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement); queryStatement = parser(queryStatement.getViewQueryParam(), queryStatement);
} else { } else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery()); queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement); queryStatement = parser(queryStatement);
} }
if (Strings.isNullOrEmpty(queryStatement.getSql()) if (Strings.isNullOrEmpty(queryStatement.getSql())
@@ -62,37 +61,37 @@ public class DefaultQueryParser implements QueryParser {
queryStatement.setSql(querySql); queryStatement.setSql(querySql);
} }
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) { public QueryStatement parser(ViewQueryParam viewQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq); log.info("parser MetricReq [{}] ", viewQueryParam);
try { try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) { if (!CollectionUtils.isEmpty(viewQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>(); List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1; Boolean isSingleTable = viewQueryParam.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) { for (MetricTable metricTable : viewQueryParam.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement); QueryStatement tableSql = parserSql(metricTable, isSingleTable, viewQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql()) if (isSingleTable && Objects.nonNull(tableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) { && !tableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql()); queryStatement.setSql(tableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq); queryStatement.setViewQueryParam(viewQueryParam);
return queryStatement; return queryStatement;
} }
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()}); tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()});
} }
if (!tables.isEmpty()) { if (!tables.isEmpty()) {
String sql = ""; String sql = "";
if (parseSqlReq.isSupportWith()) { if (viewQueryParam.isSupportWith()) {
sql = "with " + String.join(",", sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect( tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql(); Collectors.toList())) + "\n" + viewQueryParam.getSql();
} else { } else {
sql = parseSqlReq.getSql(); sql = viewQueryParam.getSql();
for (String[] tb : tables) { for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0], sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1); "(" + tb[1] + ") " + (viewQueryParam.isWithAlias() ? "" : tb[0]), -1);
} }
} }
queryStatement.setSql(sql); queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq); queryStatement.setViewQueryParam(viewQueryParam);
return queryStatement; return queryStatement;
} }
} }
@@ -104,37 +103,38 @@ public class DefaultQueryParser implements QueryParser {
} }
public QueryStatement parser(QueryStatement queryStatement) { public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery())); return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
} }
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) { public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg); log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryParam, isAgg);
try { try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg); return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) { } catch (Exception e) {
queryStatement.setErrMsg(e.getMessage()); queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e); log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
} }
return queryStatement; return queryStatement;
} }
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq, private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
ViewQueryParam viewQueryParam,
QueryStatement queryStatement) throws Exception { QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq(); MetricQueryParam metricReq = new MetricQueryParam();
metricReq.setMetrics(metricTable.getMetrics()); metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions()); metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
QueryStatement tableSql = new QueryStatement(); QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false); tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq); tableSql.setMetricQueryParam(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setViewId(queryStatement.getViewId()); tableSql.setViewId(queryStatement.getViewId());
tableSql.setSemanticModel(queryStatement.getSemanticModel()); tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) { if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql()); tableSql.setViewSql(viewQueryParam.getSql());
tableSql.setViewAlias(metricTable.getAlias()); tableSql.setViewAlias(metricTable.getAlias());
} }
tableSql = parser(tableSql, metricTable.getAggOption()); tableSql = parser(tableSql, metricTable.getAggOption());

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
@@ -23,13 +23,13 @@ public class CalciteSqlParser implements SqlParser {
@Override @Override
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception { public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
MetricQueryReq metricReq = queryStatement.getMetricReq(); MetricQueryParam metricReq = queryStatement.getMetricQueryParam();
SemanticModel semanticModel = queryStatement.getSemanticModel(); SemanticModel semanticModel = queryStatement.getSemanticModel();
if (semanticModel == null) { if (semanticModel == null) {
queryStatement.setErrMsg("semanticSchema not found"); queryStatement.setErrMsg("semanticSchema not found");
return queryStatement; return queryStatement;
} }
queryStatement.setMetricReq(metricReq); queryStatement.setMetricQueryParam(metricReq);
SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement);
AggPlanner aggBuilder = new AggPlanner(semanticSchema); AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, isAgg); aggBuilder.explain(queryStatement, isAgg);

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -17,6 +16,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRend
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender;
import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedList; import java.util.LinkedList;
@@ -35,7 +35,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
@Slf4j @Slf4j
public class AggPlanner implements Planner { public class AggPlanner implements Planner {
private MetricQueryReq metricReq; private MetricQueryParam metricReq;
private SemanticSchema schema; private SemanticSchema schema;
private SqlValidatorScope scope; private SqlValidatorScope scope;
private Stack<TableView> dataSets = new Stack<>(); private Stack<TableView> dataSets = new Stack<>();
@@ -101,7 +101,7 @@ public class AggPlanner implements Planner {
@Override @Override
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception { public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricReq = queryStatement.getMetricReq(); this.metricReq = queryStatement.getMetricQueryParam();
if (metricReq.getMetrics() == null) { if (metricReq.getMetrics() == null) {
metricReq.setMetrics(new ArrayList<>()); metricReq.setMetrics(new ArrayList<>());
} }

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MeasureNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode;
@@ -110,6 +110,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build()); return SemanticNode.buildAs(alias, tableView.build());
} }
public abstract void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public abstract void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception; SemanticSchema schema, boolean nonAgg) throws Exception;
} }

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -154,7 +154,7 @@ public class DataSourceNode extends SemanticNode {
return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
} }
public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand, public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand,
Set<String> queryDimension, List<String> measures) { Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream() queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d) .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d)
@@ -166,7 +166,7 @@ public class DataSourceNode extends SemanticNode {
} }
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand, public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand,
Set<String> queryDimension, List<String> measures, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -191,7 +191,7 @@ public class DataSourceNode extends SemanticNode {
} }
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema, public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema,
MetricQueryReq metricCommand) throws Exception { MetricQueryParam metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>(); List<DataSource> dataSources = new ArrayList<>();
// check by metric // check by metric
@@ -265,7 +265,7 @@ public class DataSourceNode extends SemanticNode {
Set<String> queryDimension, Set<String> queryDimension,
List<String> measures, List<String> measures,
Set<String> dimension, Set<String> dimension,
MetricQueryReq metricCommand, MetricQueryParam metricCommand,
SqlValidatorScope scope, SqlValidatorScope scope,
EngineType engineType) throws Exception { EngineType engineType) throws Exception {
boolean isAllMatch = true; boolean isAllMatch = true;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
@@ -29,7 +29,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
public class FilterRender extends Renderer { public class FilterRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView; TableView tableView = super.tableView;
SqlNode filterNode = null; SqlNode filterNode = null;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.parser.calcite.sql.render; package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
@@ -49,7 +49,7 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer { public class JoinRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere(); String queryWhere = metricCommand.getWhere();
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
@@ -25,7 +25,7 @@ import org.springframework.util.CollectionUtils;
public class OutputRender extends Renderer { public class OutputRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView; TableView selectDataSet = super.tableView;
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
@@ -317,9 +317,9 @@ public class SourceRender extends Renderer {
} }
} }
public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryParam metricQueryParam, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryReq.getWhere(); String queryWhere = metricQueryParam.getWhere();
Set<String> whereFields = new HashSet<>(); Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>(); List<String> fieldWhere = new ArrayList<>();
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -330,13 +330,13 @@ public class SourceRender extends Renderer {
} }
if (dataSources.size() == 1) { if (dataSources.size() == 1) {
DataSource dataSource = dataSources.get(0); DataSource dataSource = dataSources.get(0);
super.tableView = renderOne("", fieldWhere, metricQueryReq.getMetrics(), super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryReq.getDimensions(), metricQueryParam.getDimensions(),
metricQueryReq.getWhere(), dataSource, scope, schema, nonAgg); metricQueryParam.getWhere(), dataSource, scope, schema, nonAgg);
return; return;
} }
JoinRender joinRender = new JoinRender(); JoinRender joinRender = new JoinRender();
joinRender.render(metricQueryReq, dataSources, scope, schema, nonAgg); joinRender.render(metricQueryParam, dataSources, scope, schema, nonAgg);
super.tableView = joinRender.getTableView(); super.tableView = joinRender.getTableView();
} }

263
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -5,23 +5,22 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/** /**
* supplement the QueryStatement when query with custom aggregation method * supplement the QueryStatement when query with custom aggregation method
@@ -39,37 +38,37 @@ public class CalculateAggConverter implements HeadlessConverter {
public interface EngineSql { public interface EngineSql {
String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql); String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
} }
public ParseSqlReq generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version) public ViewQueryParam generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
throws Exception { throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
// 同环比 // 同环比
if (isRatioAccept(queryStructReq)) { if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version); return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
} }
ParseSqlReq sqlCommand = new ParseSqlReq(); ViewQueryParam sqlCommand = new ViewQueryParam();
String metricTableName = "v_metric_tb_tmp"; String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable(); MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName); metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null); String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommand, complete where:{}", where); log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructReq), String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryParam),
metricTableName, metricTableName,
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq), sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryStructReq)); sqlGenerateUtils.getLimit(queryParam));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false); sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructReq), sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryParam),
metricTableName, metricTableName,
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq), sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryStructReq)); sqlGenerateUtils.getLimit(queryParam));
} }
sqlCommand.setSql(sql); sqlCommand.setSql(sql);
return sqlCommand; return sqlCommand;
@@ -77,19 +76,19 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false; return false;
} }
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
if (queryStructCmd.getQueryType().isNativeAggQuery()) { if (queryParam.getQueryType().isNativeAggQuery()) {
return false; return false;
} }
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) { if (CollectionUtils.isEmpty(queryParam.getAggregators())) {
return false; return false;
} }
int nonSumFunction = 0; int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) { for (Aggregator agg : queryParam.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) { if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false; return false;
} }
@@ -105,22 +104,22 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); ViewQueryParam sqlCommend = queryStatement.getViewQueryParam();
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getSemanticModel().getDatabase();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, ViewQueryParam viewQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setSql(viewQueryParam.getSql());
sqlCommend.setTables(parseSqlReq.getTables()); sqlCommend.setTables(viewQueryParam.getTables());
sqlCommend.setVariables(parseSqlReq.getVariables()); sqlCommend.setVariables(viewQueryParam.getVariables());
sqlCommend.setSupportWith(parseSqlReq.isSupportWith()); sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
} }
/** /**
* Ratio * Ratio
*/ */
public boolean isRatioAccept(QueryStructReq queryStructCmd) { public boolean isRatioAccept(QueryParam queryParam) {
Long ratioFuncNum = queryStructCmd.getAggregators().stream() Long ratioFuncNum = queryParam.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc() .filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc()
.equals(AggOperatorEnum.RATIO_OVER))).count(); .equals(AggOperatorEnum.RATIO_OVER))).count();
if (ratioFuncNum > 0) { if (ratioFuncNum > 0) {
@@ -129,28 +128,28 @@ public class CalculateAggConverter implements HeadlessConverter {
return false; return false;
} }
public ParseSqlReq generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, public ViewQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version) String version)
throws Exception { throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
check(queryStructReq); check(queryParam);
queryStatement.setEnableOptimize(false); queryStatement.setEnableOptimize(false);
ParseSqlReq sqlCommand = new ParseSqlReq(); ViewQueryParam sqlCommand = new ViewQueryParam();
String metricTableName = "v_metric_tb_tmp"; String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable(); MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName); metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null); String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryStructReq); boolean isOver = isOverRatio(queryParam);
String sql = ""; String sql = "";
switch (engineTypeEnum) { switch (engineTypeEnum) {
case H2: case H2:
sql = new H2EngineSql().sql(queryStructReq, isOver, true, metricTableName); sql = new H2EngineSql().sql(queryParam, isOver, true, metricTableName);
break; break;
case MYSQL: case MYSQL:
case DORIS: case DORIS:
@@ -159,9 +158,9 @@ public class CalculateAggConverter implements HeadlessConverter {
sqlCommand.setSupportWith(false); sqlCommand.setSupportWith(false);
} }
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) { if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql = new MysqlEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName); sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName);
} else { } else {
sql = new CkEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName); sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName);
} }
break; break;
default: default:
@@ -172,8 +171,8 @@ public class CalculateAggConverter implements HeadlessConverter {
public class H2EngineSql implements EngineSql { public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) { public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> { String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
@@ -182,39 +181,39 @@ public class CalculateAggConverter implements HeadlessConverter {
return f.getColumn(); return f.getColumn();
} }
}).collect(Collectors.joining(",")); }).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr; : String.join(",", queryParam.getGroups()) + "," + aggStr;
} }
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) { public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) { if (Objects.nonNull(queryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-"; String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return "day," + (isOver ? addStr + "7" : addStr + "1"); return "day," + (isOver ? addStr + "7" : addStr + "1");
} }
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? "month," + addStr + "1" : "day," + addStr + "7"; return isOver ? "month," + addStr + "1" : "day," + addStr + "7";
} }
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? "year," + addStr + "1" : "month," + addStr + "1"; return isOver ? "year," + addStr + "1" : "month," + addStr + "1";
} }
} }
return ""; return "";
} }
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) { public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd); String timeDim = getTimeDim(queryParam);
String timeSpan = getTimeSpan(queryStructCmd, isOver, true); String timeSpan = getTimeSpan(queryParam, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> { String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) { if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format( return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ", "%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim); aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
} }
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ", return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryStructCmd, isOver, false), aliasLeft + timeDim, aliasRight + timeDim); getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim, aliasRight + timeDim);
} }
return String.format("%s = TIMESTAMPADD(%s,%s) ", return String.format("%s = TIMESTAMPADD(%s,%s) ",
aliasLeft + timeDim, timeSpan, aliasRight + timeDim); aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
@@ -223,7 +222,7 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
}).collect(Collectors.joining(" and ")); }).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>(); List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) { for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) { if (group.equalsIgnoreCase(timeDim)) {
continue; continue;
} }
@@ -234,31 +233,31 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
@Override @Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) { public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql = String.format( String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."), getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql, getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."), getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd)); getOrderBy(queryParam), getLimit(queryParam));
return sql; return sql;
} }
} }
public class CkEngineSql extends MysqlEngineSql { public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) { public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd); String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true); String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> { String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) { if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ", return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan); aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
} }
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s", return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim); aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim);
} }
return String.format("%s = date_add(%s,%s) ", return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan); aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
@@ -267,7 +266,7 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
}).collect(Collectors.joining(" and ")); }).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>(); List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) { for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) { if (group.equalsIgnoreCase(timeDim)) {
continue; continue;
} }
@@ -278,45 +277,45 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
@Override @Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) { public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
if (!asWith) { if (!asWith) {
return String.format( return String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."), getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql, getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."), getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd)); getOrderBy(queryParam), getLimit(queryParam));
} }
return String.format( return String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s " ",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ", + "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."), metricSql, metricSql, getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryStructCmd, isOver, "t0.", "t1."), getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd)); getOrderBy(queryParam), getLimit(queryParam));
} }
} }
public class MysqlEngineSql implements EngineSql { public class MysqlEngineSql implements EngineSql {
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) { public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) { if (Objects.nonNull(queryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-"; String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return isOver ? addStr + "7 day" : addStr + "1 day"; return isOver ? addStr + "7 day" : addStr + "1 day";
} }
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? addStr + "1 month" : addStr + "7 day"; return isOver ? addStr + "1 month" : addStr + "7 day";
} }
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) { if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? addStr + "1 year" : addStr + "1 month"; return isOver ? addStr + "1 year" : addStr + "1 month";
} }
} }
return ""; return "";
} }
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) { public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> { String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format( return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", "if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
@@ -326,22 +325,22 @@ public class CalculateAggConverter implements HeadlessConverter {
return f.getColumn(); return f.getColumn();
} }
}).collect(Collectors.joining(",")); }).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr; : String.join(",", queryParam.getGroups()) + "," + aggStr;
} }
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) { public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd); String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true); String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> { String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) { if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ", return String.format("%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan); aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
} }
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) { if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s", return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim); aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim);
} }
return String.format("%s = date_add(%s,%s) ", return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan); aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
@@ -350,7 +349,7 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
}).collect(Collectors.joining(" and ")); }).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>(); List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) { for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) { if (group.equalsIgnoreCase(timeDim)) {
continue; continue;
} }
@@ -361,24 +360,24 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
@Override @Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) { public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql = String.format( String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."), getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql, getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."), getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd)); getOrderBy(queryParam), getLimit(queryParam));
return sql; return sql;
} }
} }
private String getAllJoinSelect(QueryStructReq queryStructCmd, String alias) { private String getAllJoinSelect(QueryParam queryParam, String alias) {
String aggStr = queryStructCmd.getAggregators().stream() String aggStr = queryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") .map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "")
+ "_roll") + "_roll")
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
List<String> groups = new ArrayList<>(); List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) { for (String group : queryParam.getGroups()) {
groups.add(alias + group + " as " + group + "_roll"); groups.add(alias + group + " as " + group + "_roll");
} }
return CollectionUtils.isEmpty(groups) ? aggStr return CollectionUtils.isEmpty(groups) ? aggStr
@@ -386,29 +385,29 @@ public class CalculateAggConverter implements HeadlessConverter {
} }
private String getGroupDimWithOutTime(QueryStructReq queryStructCmd) { private String getGroupDimWithOutTime(QueryParam queryParam) {
String timeDim = getTimeDim(queryStructCmd); String timeDim = getTimeDim(queryParam);
return queryStructCmd.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
} }
private static String getTimeDim(QueryStructReq queryStructCmd) { private static String getTimeDim(QueryParam queryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class); DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(queryStructCmd.getDateInfo()); return dateModeUtils.getSysDateCol(queryParam.getDateInfo());
} }
private static String getLimit(QueryStructReq queryStructCmd) { private static String getLimit(QueryParam queryParam) {
if (queryStructCmd != null && queryStructCmd.getLimit() > 0) { if (queryParam != null && queryParam.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit()); return " limit " + String.valueOf(queryParam.getLimit());
} }
return ""; return "";
} }
private String getAllSelect(QueryStructReq queryStructCmd, String alias) { private String getAllSelect(QueryParam queryParam, String alias) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> getSelectField(f, alias)) String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, queryStructCmd.getGroups()) + "," + aggStr; : alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr;
} }
private String getSelectField(final Aggregator agg, String alias) { private String getSelectField(final Aggregator agg, String alias) {
@@ -418,32 +417,32 @@ public class CalculateAggConverter implements HeadlessConverter {
return sqlGenerateUtils.getSelectField(agg); return sqlGenerateUtils.getSelectField(agg);
} }
private String getGroupBy(QueryStructReq queryStructCmd) { private String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) { if (CollectionUtils.isEmpty(queryParam.getGroups())) {
return ""; return "";
} }
return "group by " + String.join(",", queryStructCmd.getGroups()); return "group by " + String.join(",", queryParam.getGroups());
} }
private static String getOrderBy(QueryStructReq queryStructCmd) { private static String getOrderBy(QueryParam queryParam) {
return "order by " + getTimeDim(queryStructCmd) + " desc"; return "order by " + getTimeDim(queryParam) + " desc";
} }
private boolean isOverRatio(QueryStructReq queryStructCmd) { private boolean isOverRatio(QueryParam queryParam) {
Long overCt = queryStructCmd.getAggregators().stream() Long overCt = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0; return overCt > 0;
} }
private void check(QueryStructReq queryStructCmd) throws Exception { private void check(QueryParam queryParam) throws Exception {
Long ratioOverNum = queryStructCmd.getAggregators().stream() Long ratioOverNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryStructCmd.getAggregators().stream() Long ratioRollNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count(); .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) { if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together "); throw new Exception("not support over ratio and roll ratio together ");
} }
if (getTimeDim(queryStructCmd).isEmpty()) { if (getTimeDim(queryParam).isEmpty()) {
throw new Exception("miss time filter"); throw new Exception("miss time filter");
} }

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -2,15 +2,15 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j @Slf4j
@Component("DefaultDimValueConverter") @Component("DefaultDimValueConverter")
@@ -18,7 +18,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false; return false;
} }
return true; return true;
@@ -26,16 +26,16 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) { public void convert(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream() List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensions)) { if (CollectionUtils.isEmpty(dimensions)) {
return; return;
} }
log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryStructCmd); log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryParam);
//add dimension default value to filter //add dimension default value to filter
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream() List<String> dimensionFilterBizName = queryParam.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList()); .map(Filter::getBizName).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionFilterBizName)) { if (!CollectionUtils.isEmpty(dimensionFilterBizName)) {
return; return;
@@ -46,7 +46,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
filter.setValue(dimensionResp.getDefaultValues()); filter.setValue(dimensionResp.getDefaultValues());
filter.setOperator(FilterOperatorEnum.IN); filter.setOperator(FilterOperatorEnum.IN);
filter.setName(dimensionResp.getName()); filter.setName(dimensionResp.getName());
queryStructCmd.getDimensionFilters().add(filter); queryParam.getDimensionFilters().add(filter);
} }
} }

47
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
View File

@@ -2,20 +2,19 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param; import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/** /**
* HeadlessConverter default implement * HeadlessConverter default implement
*/ */
@@ -35,7 +34,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false; return false;
} }
return !calculateConverterAgg.accept(queryStatement); return !calculateConverterAgg.accept(queryStatement);
@@ -43,38 +42,38 @@ public class ParserDefaultConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void convert(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement); MetricQueryParam metricReq = generateSqlCommand(queryStatement.getQueryParam(), queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, null)); queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null));
BeanUtils.copyProperties(metricReq, metricQueryReq); BeanUtils.copyProperties(metricReq, metricQueryParam);
} }
public MetricQueryReq generateSqlCommand(QueryStructReq queryStructReq, QueryStatement queryStatement) { public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement queryStatement) {
MetricQueryReq metricQueryReq = new MetricQueryReq(); MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryReq.setMetrics(queryStructReq.getMetrics()); metricQueryParam.setMetrics(queryParam.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups()); metricQueryParam.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null); String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where); metricQueryParam.setWhere(where);
metricQueryReq.setOrder(queryStructReq.getOrders().stream() metricQueryParam.setOrder(queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList())); .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
metricQueryReq.setVariables(queryStructReq.getParams().stream() metricQueryParam.setVariables(queryParam.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
metricQueryReq.setLimit(queryStructReq.getLimit()); metricQueryParam.setLimit(queryParam.getLimit());
// support detail query // support detail query
if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) { if (queryParam.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataSource> modelMap = queryStatement.getSemanticModel().getModelMap(); Map<Long, DataSource> modelMap = queryStatement.getSemanticModel().getModelMap();
for (Long modelId : modelMap.keySet()) { for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName(); String modelBizName = modelMap.get(modelId).getName();
String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);
metricQueryReq.getMetrics().add(internalMetricName); metricQueryParam.getMetrics().add(internalMetricName);
} }
} }
return metricQueryReq; return metricQueryParam;
} }
} }

21
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQueryOptimizer.java
View File

@@ -1,15 +1,14 @@
package com.tencent.supersonic.headless.core.planner; package com.tencent.supersonic.headless.core.planner;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/** /**
* Remove the default metric added by the system when the query only has dimensions * Remove the default metric added by the system when the query only has dimensions
*/ */
@@ -19,16 +18,16 @@ public class DetailQueryOptimizer implements QueryOptimizer {
@Override @Override
public void rewrite(QueryStatement queryStatement) { public void rewrite(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); QueryParam queryParam = queryStatement.getQueryParam();
String sqlRaw = queryStatement.getSql().trim(); String sqlRaw = queryStatement.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) { if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null"); throw new RuntimeException("sql is empty or null");
} }
log.debug("before handleNoMetric, sql:{}", sqlRaw); log.debug("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryStructCmd)) { if (isDetailQuery(queryParam)) {
if (queryStructCmd.getMetrics().size() == 0 && !CollectionUtils.isEmpty(queryStructCmd.getGroups())) { if (queryParam.getMetrics().size() == 0 && !CollectionUtils.isEmpty(queryParam.getGroups())) {
String sqlForm = "select %s from ( %s ) src_no_metric"; String sqlForm = "select %s from ( %s ) src_no_metric";
String sql = String.format(sqlForm, queryStructCmd.getGroups().stream().collect( String sql = String.format(sqlForm, queryParam.getGroups().stream().collect(
Collectors.joining(",")), sqlRaw); Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql); queryStatement.setSql(sql);
} }
@@ -36,8 +35,8 @@ public class DetailQueryOptimizer implements QueryOptimizer {
log.debug("after handleNoMetric, sql:{}", queryStatement.getSql()); log.debug("after handleNoMetric, sql:{}", queryStatement.getSql());
} }
public boolean isDetailQuery(QueryStructReq queryStructCmd) { public boolean isDetailQuery(QueryParam queryParam) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery() return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryStructCmd.getMetrics()); && CollectionUtils.isEmpty(queryParam.getMetrics());
} }
} }

19
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java Normal file
View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class MetricQueryParam {
private List<String> metrics;
private List<String> dimensions;
private Map<String, String> variables;
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,8 +1,6 @@
package com.tencent.supersonic.headless.core.pojo; package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import lombok.Data; import lombok.Data;
@@ -19,9 +17,9 @@ public class QueryStatement {
private String sourceId = ""; private String sourceId = "";
private String errMsg = ""; private String errMsg = "";
private Boolean ok; private Boolean ok;
private QueryStructReq queryStructReq; private QueryParam queryParam;
private MetricQueryReq metricReq; private MetricQueryParam metricQueryParam;
private ParseSqlReq parseSqlReq; private ViewQueryParam viewQueryParam;
private Integer status = 0; private Integer status = 0;
private Boolean isS2SQL = false; private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges; private List<ImmutablePair<String, String>> timeRanges;

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class ViewQueryParam {
private Map<String, String> variables;
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;
private boolean withAlias = true;
public Map<String, String> getVariables() {
if (variables == null) {
variables = new HashMap<>();
}
return variables;
}
}

110
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -1,5 +1,12 @@
package com.tencent.supersonic.headless.core.utils; package com.tencent.supersonic.headless.core.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -10,23 +17,15 @@ import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Collections; import java.util.Collections;
@@ -38,13 +37,14 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import static com.tencent.supersonic.common.pojo.Constants.DAY; import org.apache.commons.lang3.StringUtils;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; import org.apache.commons.lang3.tuple.ImmutablePair;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; import org.apache.commons.lang3.tuple.Triple;
import static com.tencent.supersonic.common.pojo.Constants.MONTH; import org.apache.logging.log4j.util.Strings;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; import org.springframework.beans.factory.annotation.Value;
import static com.tencent.supersonic.common.pojo.Constants.WEEK; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/** /**
* tools functions to analyze queryStructReq * tools functions to analyze queryStructReq
@@ -94,25 +94,25 @@ public class SqlGenerateUtils {
return selectSql; return selectSql;
} }
public String getLimit(QueryStructReq queryStructCmd) { public String getLimit(QueryParam queryParam) {
if (queryStructCmd.getLimit() > 0) { if (queryParam.getLimit() > 0) {
return " limit " + queryStructCmd.getLimit(); return " limit " + queryParam.getLimit();
} }
return ""; return "";
} }
public String getSelect(QueryStructReq queryStructCmd) { public String getSelect(QueryParam queryParam) {
String aggStr = queryStructCmd.getAggregators().stream().map(this::getSelectField) String aggStr = queryParam.getAggregators().stream().map(this::getSelectField)
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr; : String.join(",", queryParam.getGroups()) + "," + aggStr;
} }
public String getSelect(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) { public String getSelect(QueryParam queryParam, Map<String, String> deriveMetrics) {
String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics)) String aggStr = queryParam.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr; : String.join(",", queryParam.getGroups()) + "," + aggStr;
} }
public String getSelectField(final Aggregator agg) { public String getSelectField(final Aggregator agg) {
@@ -134,42 +134,42 @@ public class SqlGenerateUtils {
return deriveMetrics.get(agg.getColumn()); return deriveMetrics.get(agg.getColumn());
} }
public String getGroupBy(QueryStructReq queryStructCmd) { public String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) { if (CollectionUtils.isEmpty(queryParam.getGroups())) {
return ""; return "";
} }
return "group by " + String.join(",", queryStructCmd.getGroups()); return "group by " + String.join(",", queryParam.getGroups());
} }
public String getOrderBy(QueryStructReq queryStructCmd) { public String getOrderBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) { if (CollectionUtils.isEmpty(queryParam.getOrders())) {
return ""; return "";
} }
return "order by " + queryStructCmd.getOrders().stream() return "order by " + queryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ") .map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
} }
public String getOrderBy(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) { public String getOrderBy(QueryParam queryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) { if (CollectionUtils.isEmpty(queryParam.getOrders())) {
return ""; return "";
} }
if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { if (!queryParam.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryStructCmd); return getOrderBy(queryParam);
} }
return "order by " + queryStructCmd.getOrders().stream() return "order by " + queryParam.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn()) .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ") : order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(",")); .collect(Collectors.joining(","));
} }
public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) { public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter()); String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp); String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryStructReq, whereClauseFromFilter, whereFromDate); return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate);
} }
private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter, private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
String whereFromDate) { String whereFromDate) {
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
@@ -179,7 +179,7 @@ public class SqlGenerateUtils {
return whereFromDate; return whereFromDate;
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) { } else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
log.info("the current date information is empty, enter the date initialization logic"); log.info("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo()); return dateModeUtils.defaultRecentDateInfo(queryParam.getDateInfo());
} }
return whereClauseFromFilter; return whereClauseFromFilter;
} }
@@ -204,11 +204,11 @@ public class SqlGenerateUtils {
return dateModeUtils.getDateWhereStr(dateInfo, dateDate); return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
} }
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd, ItemDateResp dataDate) { public Triple<String, String, String> getBeginEndTime(QueryParam queryParam, ItemDateResp dataDate) {
if (Objects.isNull(queryStructCmd.getDateInfo())) { if (Objects.isNull(queryParam.getDateInfo())) {
return Triple.of("", "", ""); return Triple.of("", "", "");
} }
DateConf dateConf = queryStructCmd.getDateInfo(); DateConf dateConf = queryParam.getDateInfo();
String dateInfo = dateModeUtils.getSysDateCol(dateConf); String dateInfo = dateModeUtils.getSysDateCol(dateConf);
if (dateInfo.isEmpty()) { if (dateInfo.isEmpty()) {
return Triple.of("", "", ""); return Triple.of("", "", "");
@@ -272,13 +272,13 @@ public class SqlGenerateUtils {
} }
public String generateDerivedMetric(final List<MetricSchemaResp> metricResps, final Set<String> allFields, public String generateDerivedMetric(final List<MetricSchemaResp> metricResps, final Set<String> allFields,
final Map<String, Measure> allMeasures, final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps, final List<DimSchemaResp> dimensionResps,
final String expression, final MetricDefineType metricDefineType, final String expression, final MetricDefineType metricDefineType,
AggOption aggOption, AggOption aggOption,
Set<String> visitedMetric, Set<String> visitedMetric,
Set<String> measures, Set<String> measures,
Set<String> dimensions) { Set<String> dimensions) {
Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression); Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression);
if (!CollectionUtils.isEmpty(fields)) { if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>(); Map<String, String> replace = new HashMap<>();

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
View File

@@ -32,10 +32,10 @@ public class MetricDrillDownChecker {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs(); Object[] objects = joinPoint.getArgs();
QueryStatement queryStatement = (QueryStatement) objects[0]; QueryStatement queryStatement = (QueryStatement) objects[0];
if (queryStatement.getParseSqlReq() == null) { if (queryStatement.getViewQueryParam() == null) {
return joinPoint.proceed(); return joinPoint.proceed();
} }
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getParseSqlReq().getSql()); checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getViewQueryParam().getSql());
return joinPoint.proceed(); return joinPoint.proceed();
} }

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item; import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
@@ -48,18 +49,17 @@ import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@Service @Service
@@ -165,10 +165,13 @@ public class QueryServiceImpl implements QueryService {
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq); SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq); QueryParam queryParam = new QueryParam();
queryReqConverter.convert(queryStructReq, queryParam);
//queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setQueryParam(queryParam);
queryStatement.setIsS2SQL(false); queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setViewId(queryStatement.getQueryStructReq().getViewId()); queryStatement.setViewId(queryStructReq.getViewId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp); queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp); SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel); queryStatement.setSemanticModel(semanticModel);
@@ -181,7 +184,7 @@ public class QueryServiceImpl implements QueryService {
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq); QueryStatement queryStatement = buildQueryStatement(queryStructReq);
SemanticModel semanticModel = queryStatement.getSemanticModel(); SemanticModel semanticModel = queryStatement.getSemanticModel();
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); queryStatement.setModelIds(queryStructReq.getModelIds());
queryStatement.setSemanticModel(semanticModel); queryStatement.setSemanticModel(semanticModel);
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement = plan(queryStatement); queryStatement = plan(queryStatement);

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -11,11 +11,12 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq; import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -27,15 +28,6 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@@ -45,6 +37,14 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component @Component
@Slf4j @Slf4j
@@ -60,7 +60,7 @@ public class QueryReqConverter {
private SqlGenerateUtils sqlGenerateUtils; private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QuerySqlReq querySQLReq, public QueryStatement convert(QuerySqlReq querySQLReq,
SemanticSchemaResp semanticSchemaResp) throws Exception { SemanticSchemaResp semanticSchemaResp) throws Exception {
if (semanticSchemaResp == null) { if (semanticSchemaResp == null) {
return new QueryStatement(); return new QueryStatement();
@@ -103,7 +103,7 @@ public class QueryReqConverter {
List<MetricTable> tables = new ArrayList<>(); List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable); tables.add(metricTable);
//4.build ParseSqlReq //4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq(); ViewQueryParam result = new ViewQueryParam();
BeanUtils.copyProperties(querySQLReq, result); BeanUtils.copyProperties(querySQLReq, result);
result.setTables(tables); result.setTables(tables);
@@ -116,13 +116,16 @@ public class QueryReqConverter {
//5. do deriveMetric //5. do deriveMetric
generateDerivedMetric(semanticSchemaResp, aggOption, result); generateDerivedMetric(semanticSchemaResp, aggOption, result);
//6.physicalSql by ParseSqlReq //6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setViewId(querySQLReq.getViewId()); queryStructReq.setViewId(querySQLReq.getViewId());
queryStructReq.setQueryType(getQueryType(aggOption)); queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
convert(queryStructReq, queryParam);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq); queryStatement.setQueryParam(queryParam);
queryStatement.setParseSqlReq(result); queryStatement.setViewQueryParam(result);
queryStatement.setIsS2SQL(true); queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(querySQLReq.getViewId()); queryStatement.setViewId(querySQLReq.getViewId());
@@ -131,6 +134,13 @@ public class QueryReqConverter {
return queryStatement; return queryStatement;
} }
public void convert(QueryStructReq queryStructReq, QueryParam queryParam) {
BeanUtils.copyProperties(queryStructReq, queryParam);
queryParam.setOrders(queryStructReq.getOrders());
queryParam.setMetrics(queryStructReq.getMetrics());
queryParam.setGroups(queryStructReq.getGroups());
}
private AggOption getAggOption(QuerySqlReq databaseReq) { private AggOption getAggOption(QuerySqlReq databaseReq) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
@@ -229,9 +239,9 @@ public class QueryReqConverter {
} }
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
ParseSqlReq parseSqlReq) { ViewQueryParam viewQueryParam) {
String sql = parseSqlReq.getSql(); String sql = viewQueryParam.getSql();
for (MetricTable metricTable : parseSqlReq.getTables()) { for (MetricTable metricTable : viewQueryParam.getTables()) {
List<String> measures = new ArrayList<>(); List<String> measures = new ArrayList<>();
Map<String, String> replaces = new HashMap<>(); Map<String, String> replaces = new HashMap<>();
generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(), generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(),
@@ -247,7 +257,7 @@ public class QueryReqConverter {
} }
} }
} }
parseSqlReq.setSql(sql); viewQueryParam.setSql(sql);
} }
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,

18
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -3,11 +3,12 @@ package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
@@ -15,21 +16,20 @@ import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
class HeadlessParserServiceTest { class HeadlessParserServiceTest {
private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>(); private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>();
public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricQueryReq, boolean isAgg) { public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryParam metricQueryParam,
boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp(); SqlParserResp sqlParser = new SqlParserResp();
try { try {
if (semanticSchema == null) { if (semanticSchema == null) {
@@ -38,14 +38,14 @@ class HeadlessParserServiceTest {
} }
AggPlanner aggBuilder = new AggPlanner(semanticSchema); AggPlanner aggBuilder = new AggPlanner(semanticSchema);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricReq(metricQueryReq); queryStatement.setMetricQueryParam(metricQueryParam);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType)); sqlParser.setSql(aggBuilder.getSql(engineType));
sqlParser.setSourceId(aggBuilder.getSourceId()); sqlParser.setSourceId(aggBuilder.getSourceId());
} catch (Exception e) { } catch (Exception e) {
sqlParser.setErrMsg(e.getMessage()); sqlParser.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e); log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
} }
return sqlParser; return sqlParser;
} }
@@ -162,7 +162,7 @@ class HeadlessParserServiceTest {
//HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); //HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
MetricQueryReq metricCommand = new MetricQueryReq(); MetricQueryParam metricCommand = new MetricQueryParam();
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
@@ -174,7 +174,7 @@ class HeadlessParserServiceTest {
addDepartment(semanticSchema); addDepartment(semanticSchema);
MetricQueryReq metricCommand2 = new MetricQueryReq(); MetricQueryParam metricCommand2 = new MetricQueryParam();
metricCommand2.setDimensions(new ArrayList<>( metricCommand2.setDimensions(new ArrayList<>(
Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));