topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:38:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) code refactor (#724)

Browse Source
This commit is contained in:
jipeli
2024-02-07 14:28:01 +08:00
committed by GitHub
parent 9600456bae
commit 329ad327b0
22 changed files with 438 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java Normal file
View File

@@ -0,0 +1,42 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.Data;
@Data
public class QueryParam {
// struct
private List<String> groups = new ArrayList();
private List<Aggregator> aggregators = new ArrayList();
private List<Order> orders = new ArrayList();
private List<Filter> dimensionFilters = new ArrayList();
private List<Filter> metricFilters = new ArrayList();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType;
private String s2SQL;
private String correctS2SQL;
private Long viewId;
private String viewName;
private Set<Long> modelIds = new HashSet<>();
private List<Param> params = new ArrayList<>();
// metric
private List<String> metrics = new ArrayList();
private List<String> dimensions;
private Map<String, String> variables;
private String where;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

88
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java
View File

@@ -2,52 +2,51 @@ package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
@Slf4j
@Primary
public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getViewQueryParam())) {
queryStatement.setViewQueryParam(new ViewQueryParam());
}
if (Objects.isNull(queryStatement.getMetricReq())) {
queryStatement.setMetricReq(new MetricQueryReq());
if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricQueryParam(new MetricQueryParam());
}
log.info("SemanticConverter before [{}]", queryStructReq);
log.info("SemanticConverter before [{}]", queryParam);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getViewQueryParam(),
queryStatement.getMetricQueryParam());
if (!queryStatement.getViewQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getViewQueryParam(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
if (Strings.isNullOrEmpty(queryStatement.getSql())
@@ -62,37 +61,37 @@ public class DefaultQueryParser implements QueryParser {
queryStatement.setSql(querySql);
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
public QueryStatement parser(ViewQueryParam viewQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", viewQueryParam);
try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
if (!CollectionUtils.isEmpty(viewQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq);
Boolean isSingleTable = viewQueryParam.getTables().size() == 1;
for (MetricTable metricTable : viewQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable, viewQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(tableSql.getViewSimplifySql())
&& !tableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(tableSql.getViewSimplifySql());
queryStatement.setViewQueryParam(viewQueryParam);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()});
tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (parseSqlReq.isSupportWith()) {
if (viewQueryParam.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql();
Collectors.toList())) + "\n" + viewQueryParam.getSql();
} else {
sql = parseSqlReq.getSql();
sql = viewQueryParam.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
"(" + tb[1] + ") " + (viewQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq);
queryStatement.setViewQueryParam(viewQueryParam);
return queryStatement;
}
}
@@ -104,37 +103,38 @@ public class DefaultQueryParser implements QueryParser {
}
public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery()));
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryParam, isAgg);
try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq,
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
ViewQueryParam viewQueryParam,
QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq();
MetricQueryParam metricReq = new MetricQueryParam();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMetricQueryParam(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setViewId(queryStatement.getViewId());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewSql(viewQueryParam.getSql());
tableSql.setViewAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
@@ -23,13 +23,13 @@ public class CalciteSqlParser implements SqlParser {
@Override
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
MetricQueryReq metricReq = queryStatement.getMetricReq();
MetricQueryParam metricReq = queryStatement.getMetricQueryParam();
SemanticModel semanticModel = queryStatement.getSemanticModel();
if (semanticModel == null) {
queryStatement.setErrMsg("semanticSchema not found");
return queryStatement;
}
queryStatement.setMetricReq(metricReq);
queryStatement.setMetricQueryParam(metricReq);
SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement);
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, isAgg);

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -17,6 +16,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRend
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -35,7 +35,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
@Slf4j
public class AggPlanner implements Planner {
private MetricQueryReq metricReq;
private MetricQueryParam metricReq;
private SemanticSchema schema;
private SqlValidatorScope scope;
private Stack<TableView> dataSets = new Stack<>();
@@ -101,7 +101,7 @@ public class AggPlanner implements Planner {
@Override
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricReq = queryStatement.getMetricReq();
this.metricReq = queryStatement.getMetricQueryParam();
if (metricReq.getMetrics() == null) {
metricReq.setMetrics(new ArrayList<>());
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode;
@@ -110,6 +110,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build());
}
public abstract void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
public abstract void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception;
}

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -154,7 +154,7 @@ public class DataSourceNode extends SemanticNode {
return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
}
public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand,
public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand,
Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d)
@@ -166,7 +166,7 @@ public class DataSourceNode extends SemanticNode {
}
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand,
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryParam metricCommand,
Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -191,7 +191,7 @@ public class DataSourceNode extends SemanticNode {
}
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema,
MetricQueryReq metricCommand) throws Exception {
MetricQueryParam metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>();
// check by metric
@@ -265,7 +265,7 @@ public class DataSourceNode extends SemanticNode {
Set<String> queryDimension,
List<String> measures,
Set<String> dimension,
MetricQueryReq metricCommand,
MetricQueryParam metricCommand,
SqlValidatorScope scope,
EngineType engineType) throws Exception {
boolean isAllMatch = true;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
@@ -29,7 +29,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
public class FilterRender extends Renderer {
@Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView;
SqlNode filterNode = null;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
@@ -49,7 +49,7 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer {
@Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
@@ -25,7 +25,7 @@ import org.springframework.util.CollectionUtils;
public class OutputRender extends Renderer {
@Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView;
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
@@ -317,9 +317,9 @@ public class SourceRender extends Renderer {
}
}
public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope,
public void render(MetricQueryParam metricQueryParam, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryReq.getWhere();
String queryWhere = metricQueryParam.getWhere();
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -330,13 +330,13 @@ public class SourceRender extends Renderer {
}
if (dataSources.size() == 1) {
DataSource dataSource = dataSources.get(0);
super.tableView = renderOne("", fieldWhere, metricQueryReq.getMetrics(),
metricQueryReq.getDimensions(),
metricQueryReq.getWhere(), dataSource, scope, schema, nonAgg);
super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(),
metricQueryParam.getWhere(), dataSource, scope, schema, nonAgg);
return;
}
JoinRender joinRender = new JoinRender();
joinRender.render(metricQueryReq, dataSources, scope, schema, nonAgg);
joinRender.render(metricQueryParam, dataSources, scope, schema, nonAgg);
super.tableView = joinRender.getTableView();
}

263
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -5,23 +5,22 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* supplement the QueryStatement when query with custom aggregation method
@@ -39,37 +38,37 @@ public class CalculateAggConverter implements HeadlessConverter {
public interface EngineSql {
String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql);
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public ParseSqlReq generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
public ViewQueryParam generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
if (isRatioAccept(queryStructReq)) {
if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
ViewQueryParam sqlCommand = new ViewQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryParam),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
sqlGenerateUtils.getLimit(queryStructReq));
sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryParam),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
sqlGenerateUtils.getLimit(queryStructReq));
sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
}
sqlCommand.setSql(sql);
return sqlCommand;
@@ -77,19 +76,19 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false;
}
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
QueryParam queryParam = queryStatement.getQueryParam();
if (queryParam.getQueryType().isNativeAggQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
if (CollectionUtils.isEmpty(queryParam.getAggregators())) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
for (Aggregator agg : queryParam.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
@@ -105,22 +104,22 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
ViewQueryParam sqlCommend = queryStatement.getViewQueryParam();
Database database = queryStatement.getSemanticModel().getDatabase();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
ViewQueryParam viewQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setVariables(parseSqlReq.getVariables());
sqlCommend.setSupportWith(parseSqlReq.isSupportWith());
sqlCommend.setSql(viewQueryParam.getSql());
sqlCommend.setTables(viewQueryParam.getTables());
sqlCommend.setVariables(viewQueryParam.getVariables());
sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
}
/**
* Ratio
*/
public boolean isRatioAccept(QueryStructReq queryStructCmd) {
Long ratioFuncNum = queryStructCmd.getAggregators().stream()
public boolean isRatioAccept(QueryParam queryParam) {
Long ratioFuncNum = queryParam.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc()
.equals(AggOperatorEnum.RATIO_OVER))).count();
if (ratioFuncNum > 0) {
@@ -129,28 +128,28 @@ public class CalculateAggConverter implements HeadlessConverter {
return false;
}
public ParseSqlReq generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
public ViewQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
check(queryStructReq);
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
queryStatement.setEnableOptimize(false);
ParseSqlReq sqlCommand = new ParseSqlReq();
ViewQueryParam sqlCommand = new ViewQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryStructReq);
boolean isOver = isOverRatio(queryParam);
String sql = "";
switch (engineTypeEnum) {
case H2:
sql = new H2EngineSql().sql(queryStructReq, isOver, true, metricTableName);
sql = new H2EngineSql().sql(queryParam, isOver, true, metricTableName);
break;
case MYSQL:
case DORIS:
@@ -159,9 +158,9 @@ public class CalculateAggConverter implements HeadlessConverter {
sqlCommand.setSupportWith(false);
}
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql = new MysqlEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName);
} else {
sql = new CkEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), metricTableName);
}
break;
default:
@@ -172,8 +171,8 @@ public class CalculateAggConverter implements HeadlessConverter {
public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
@@ -182,39 +181,39 @@ public class CalculateAggConverter implements HeadlessConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) {
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return "day," + (isOver ? addStr + "7" : addStr + "1");
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? "month," + addStr + "1" : "day," + addStr + "7";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? "year," + addStr + "1" : "month," + addStr + "1";
}
}
return "";
}
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryStructCmd, isOver, false), aliasLeft + timeDim, aliasRight + timeDim);
getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim, aliasRight + timeDim);
}
return String.format("%s = TIMESTAMPADD(%s,%s) ",
aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
@@ -223,7 +222,7 @@ public class CalculateAggConverter implements HeadlessConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -234,31 +233,31 @@ public class CalculateAggConverter implements HeadlessConverter {
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) {
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam), getLimit(queryParam));
return sql;
}
}
public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim);
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
@@ -267,7 +266,7 @@ public class CalculateAggConverter implements HeadlessConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -278,45 +277,45 @@ public class CalculateAggConverter implements HeadlessConverter {
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) {
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
if (!asWith) {
return String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam), getLimit(queryParam));
}
return String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."),
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
metricSql, metricSql, getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam), getLimit(queryParam));
}
}
public class MysqlEngineSql implements EngineSql {
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) {
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return isOver ? addStr + "7 day" : addStr + "1 day";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? addStr + "1 month" : addStr + "7 day";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
if (queryParam.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? addStr + "1 year" : addStr + "1 month";
}
}
return "";
}
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
@@ -326,22 +325,22 @@ public class CalculateAggConverter implements HeadlessConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
if (queryParam.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim);
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
@@ -350,7 +349,7 @@ public class CalculateAggConverter implements HeadlessConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -361,24 +360,24 @@ public class CalculateAggConverter implements HeadlessConverter {
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql) {
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam), getLimit(queryParam));
return sql;
}
}
private String getAllJoinSelect(QueryStructReq queryStructCmd, String alias) {
String aggStr = queryStructCmd.getAggregators().stream()
private String getAllJoinSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "")
+ "_roll")
.collect(Collectors.joining(","));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
for (String group : queryParam.getGroups()) {
groups.add(alias + group + " as " + group + "_roll");
}
return CollectionUtils.isEmpty(groups) ? aggStr
@@ -386,29 +385,29 @@ public class CalculateAggConverter implements HeadlessConverter {
}
private String getGroupDimWithOutTime(QueryStructReq queryStructCmd) {
String timeDim = getTimeDim(queryStructCmd);
return queryStructCmd.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
private String getGroupDimWithOutTime(QueryParam queryParam) {
String timeDim = getTimeDim(queryParam);
return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(QueryStructReq queryStructCmd) {
private static String getTimeDim(QueryParam queryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(queryStructCmd.getDateInfo());
return dateModeUtils.getSysDateCol(queryParam.getDateInfo());
}
private static String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd != null && queryStructCmd.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit());
private static String getLimit(QueryParam queryParam) {
if (queryParam != null && queryParam.getLimit() > 0) {
return " limit " + String.valueOf(queryParam.getLimit());
}
return "";
}
private String getAllSelect(QueryStructReq queryStructCmd, String alias) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> getSelectField(f, alias))
private String getAllSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: alias + String.join("," + alias, queryStructCmd.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr;
}
private String getSelectField(final Aggregator agg, String alias) {
@@ -418,32 +417,32 @@ public class CalculateAggConverter implements HeadlessConverter {
return sqlGenerateUtils.getSelectField(agg);
}
private String getGroupBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
private String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryStructCmd.getGroups());
return "group by " + String.join(",", queryParam.getGroups());
}
private static String getOrderBy(QueryStructReq queryStructCmd) {
return "order by " + getTimeDim(queryStructCmd) + " desc";
private static String getOrderBy(QueryParam queryParam) {
return "order by " + getTimeDim(queryParam) + " desc";
}
private boolean isOverRatio(QueryStructReq queryStructCmd) {
Long overCt = queryStructCmd.getAggregators().stream()
private boolean isOverRatio(QueryParam queryParam) {
Long overCt = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0;
}
private void check(QueryStructReq queryStructCmd) throws Exception {
Long ratioOverNum = queryStructCmd.getAggregators().stream()
private void check(QueryParam queryParam) throws Exception {
Long ratioOverNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryStructCmd.getAggregators().stream()
Long ratioRollNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}
if (getTimeDim(queryStructCmd).isEmpty()) {
if (getTimeDim(queryParam).isEmpty()) {
throw new Exception("miss time filter");
}

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -2,15 +2,15 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component("DefaultDimValueConverter")
@@ -18,7 +18,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false;
}
return true;
@@ -26,16 +26,16 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
QueryParam queryParam = queryStatement.getQueryParam();
List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryStructCmd);
log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryParam);
//add dimension default value to filter
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream()
List<String> dimensionFilterBizName = queryParam.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionFilterBizName)) {
return;
@@ -46,7 +46,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
filter.setValue(dimensionResp.getDefaultValues());
filter.setOperator(FilterOperatorEnum.IN);
filter.setName(dimensionResp.getName());
queryStructCmd.getDimensionFilters().add(filter);
queryParam.getDimensionFilters().add(filter);
}
}

47
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
View File

@@ -2,20 +2,19 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* HeadlessConverter default implement
*/
@@ -35,7 +34,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false;
}
return !calculateConverterAgg.accept(queryStatement);
@@ -43,38 +42,38 @@ public class ParserDefaultConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, null));
BeanUtils.copyProperties(metricReq, metricQueryReq);
QueryParam queryParam = queryStatement.getQueryParam();
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
MetricQueryParam metricReq = generateSqlCommand(queryStatement.getQueryParam(), queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null));
BeanUtils.copyProperties(metricReq, metricQueryParam);
}
public MetricQueryReq generateSqlCommand(QueryStructReq queryStructReq, QueryStatement queryStatement) {
MetricQueryReq metricQueryReq = new MetricQueryReq();
metricQueryReq.setMetrics(queryStructReq.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement queryStatement) {
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(queryParam.getMetrics());
metricQueryParam.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where);
metricQueryReq.setOrder(queryStructReq.getOrders().stream()
metricQueryParam.setWhere(where);
metricQueryParam.setOrder(queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
metricQueryReq.setVariables(queryStructReq.getParams().stream()
metricQueryParam.setVariables(queryParam.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
metricQueryReq.setLimit(queryStructReq.getLimit());
metricQueryParam.setLimit(queryParam.getLimit());
// support detail query
if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) {
if (queryParam.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataSource> modelMap = queryStatement.getSemanticModel().getModelMap();
for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName();
String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);
metricQueryReq.getMetrics().add(internalMetricName);
metricQueryParam.getMetrics().add(internalMetricName);
}
}
return metricQueryReq;
return metricQueryParam;
}
}

21
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQueryOptimizer.java
View File

@@ -1,15 +1,14 @@
package com.tencent.supersonic.headless.core.planner;
import com.google.common.base.Strings;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* Remove the default metric added by the system when the query only has dimensions
*/
@@ -19,16 +18,16 @@ public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
QueryParam queryParam = queryStatement.getQueryParam();
String sqlRaw = queryStatement.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.debug("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryStructCmd)) {
if (queryStructCmd.getMetrics().size() == 0 && !CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
if (isDetailQuery(queryParam)) {
if (queryParam.getMetrics().size() == 0 && !CollectionUtils.isEmpty(queryParam.getGroups())) {
String sqlForm = "select %s from ( %s ) src_no_metric";
String sql = String.format(sqlForm, queryStructCmd.getGroups().stream().collect(
String sql = String.format(sqlForm, queryParam.getGroups().stream().collect(
Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql);
}
@@ -36,8 +35,8 @@ public class DetailQueryOptimizer implements QueryOptimizer {
log.debug("after handleNoMetric, sql:{}", queryStatement.getSql());
}
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryStructCmd.getMetrics());
public boolean isDetailQuery(QueryParam queryParam) {
return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryParam.getMetrics());
}
}

19
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java Normal file
View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class MetricQueryParam {
private List<String> metrics;
private List<String> dimensions;
private Map<String, String> variables;
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,8 +1,6 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import lombok.Data;
@@ -19,9 +17,9 @@ public class QueryStatement {
private String sourceId = "";
private String errMsg = "";
private Boolean ok;
private QueryStructReq queryStructReq;
private MetricQueryReq metricReq;
private ParseSqlReq parseSqlReq;
private QueryParam queryParam;
private MetricQueryParam metricQueryParam;
private ViewQueryParam viewQueryParam;
private Integer status = 0;
private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges;

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class ViewQueryParam {
private Map<String, String> variables;
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;
private boolean withAlias = true;
public Map<String, String> getVariables() {
if (variables == null) {
variables = new HashMap<>();
}
return variables;
}
}

110
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -1,5 +1,12 @@
package com.tencent.supersonic.headless.core.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -10,23 +17,15 @@ import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
@@ -38,13 +37,14 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* tools functions to analyze queryStructReq
@@ -94,25 +94,25 @@ public class SqlGenerateUtils {
return selectSql;
}
public String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd.getLimit() > 0) {
return " limit " + queryStructCmd.getLimit();
public String getLimit(QueryParam queryParam) {
if (queryParam.getLimit() > 0) {
return " limit " + queryParam.getLimit();
}
return "";
}
public String getSelect(QueryStructReq queryStructCmd) {
String aggStr = queryStructCmd.getAggregators().stream().map(this::getSelectField)
public String getSelect(QueryParam queryParam) {
String aggStr = queryParam.getAggregators().stream().map(this::getSelectField)
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getSelect(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
public String getSelect(QueryParam queryParam, Map<String, String> deriveMetrics) {
String aggStr = queryParam.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getSelectField(final Aggregator agg) {
@@ -134,42 +134,42 @@ public class SqlGenerateUtils {
return deriveMetrics.get(agg.getColumn());
}
public String getGroupBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
public String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryStructCmd.getGroups());
return "group by " + String.join(",", queryParam.getGroups());
}
public String getOrderBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
public String getOrderBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
return "";
}
return "order by " + queryStructCmd.getOrders().stream()
return "order by " + queryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String getOrderBy(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
public String getOrderBy(QueryParam queryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
return "";
}
if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryStructCmd);
if (!queryParam.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryParam);
}
return "order by " + queryStructCmd.getOrders().stream()
return "order by " + queryParam.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryStructReq, whereClauseFromFilter, whereFromDate);
public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate);
}
private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
String whereFromDate) {
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
@@ -179,7 +179,7 @@ public class SqlGenerateUtils {
return whereFromDate;
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
log.info("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
return dateModeUtils.defaultRecentDateInfo(queryParam.getDateInfo());
}
return whereClauseFromFilter;
}
@@ -204,11 +204,11 @@ public class SqlGenerateUtils {
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd, ItemDateResp dataDate) {
if (Objects.isNull(queryStructCmd.getDateInfo())) {
public Triple<String, String, String> getBeginEndTime(QueryParam queryParam, ItemDateResp dataDate) {
if (Objects.isNull(queryParam.getDateInfo())) {
return Triple.of("", "", "");
}
DateConf dateConf = queryStructCmd.getDateInfo();
DateConf dateConf = queryParam.getDateInfo();
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
if (dateInfo.isEmpty()) {
return Triple.of("", "", "");
@@ -272,13 +272,13 @@ public class SqlGenerateUtils {
}
public String generateDerivedMetric(final List<MetricSchemaResp> metricResps, final Set<String> allFields,
final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps,
final String expression, final MetricDefineType metricDefineType,
AggOption aggOption,
Set<String> visitedMetric,
Set<String> measures,
Set<String> dimensions) {
final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps,
final String expression, final MetricDefineType metricDefineType,
AggOption aggOption,
Set<String> visitedMetric,
Set<String> measures,
Set<String> dimensions) {
Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression);
if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>();

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
View File

@@ -32,10 +32,10 @@ public class MetricDrillDownChecker {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs();
QueryStatement queryStatement = (QueryStatement) objects[0];
if (queryStatement.getParseSqlReq() == null) {
if (queryStatement.getViewQueryParam() == null) {
return joinPoint.proceed();
}
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getParseSqlReq().getSql());
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getViewQueryParam().getSql());
return joinPoint.proceed();
}

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
@@ -48,18 +49,17 @@ import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@Service
@@ -165,10 +165,13 @@ public class QueryServiceImpl implements QueryService {
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
QueryParam queryParam = new QueryParam();
queryReqConverter.convert(queryStructReq, queryParam);
//queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setQueryParam(queryParam);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setViewId(queryStatement.getQueryStructReq().getViewId());
queryStatement.setViewId(queryStructReq.getViewId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
@@ -181,7 +184,7 @@ public class QueryServiceImpl implements QueryService {
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
SemanticModel semanticModel = queryStatement.getSemanticModel();
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
queryStatement.setModelIds(queryStructReq.getModelIds());
queryStatement.setSemanticModel(semanticModel);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement = plan(queryStatement);

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -11,11 +11,12 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -27,15 +28,6 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -45,6 +37,14 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@@ -60,7 +60,7 @@ public class QueryReqConverter {
private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QuerySqlReq querySQLReq,
SemanticSchemaResp semanticSchemaResp) throws Exception {
SemanticSchemaResp semanticSchemaResp) throws Exception {
if (semanticSchemaResp == null) {
return new QueryStatement();
@@ -103,7 +103,7 @@ public class QueryReqConverter {
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
ViewQueryParam result = new ViewQueryParam();
BeanUtils.copyProperties(querySQLReq, result);
result.setTables(tables);
@@ -116,13 +116,16 @@ public class QueryReqConverter {
//5. do deriveMetric
generateDerivedMetric(semanticSchemaResp, aggOption, result);
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setViewId(querySQLReq.getViewId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
convert(queryStructReq, queryParam);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setParseSqlReq(result);
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(querySQLReq.getViewId());
@@ -131,6 +134,13 @@ public class QueryReqConverter {
return queryStatement;
}
public void convert(QueryStructReq queryStructReq, QueryParam queryParam) {
BeanUtils.copyProperties(queryStructReq, queryParam);
queryParam.setOrders(queryStructReq.getOrders());
queryParam.setMetrics(queryStructReq.getMetrics());
queryParam.setGroups(queryStructReq.getGroups());
}
private AggOption getAggOption(QuerySqlReq databaseReq) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
@@ -229,9 +239,9 @@ public class QueryReqConverter {
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
ParseSqlReq parseSqlReq) {
String sql = parseSqlReq.getSql();
for (MetricTable metricTable : parseSqlReq.getTables()) {
ViewQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) {
List<String> measures = new ArrayList<>();
Map<String, String> replaces = new HashMap<>();
generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(),
@@ -247,7 +257,7 @@ public class QueryReqConverter {
}
}
}
parseSqlReq.setSql(sql);
viewQueryParam.setSql(sql);
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,

18
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -3,11 +3,12 @@ package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
@@ -15,21 +16,20 @@ import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Slf4j
class HeadlessParserServiceTest {
private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>();
public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricQueryReq, boolean isAgg) {
public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryParam metricQueryParam,
boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp();
try {
if (semanticSchema == null) {
@@ -38,14 +38,14 @@ class HeadlessParserServiceTest {
}
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricReq(metricQueryReq);
queryStatement.setMetricQueryParam(metricQueryParam);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType));
sqlParser.setSourceId(aggBuilder.getSourceId());
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
}
return sqlParser;
}
@@ -162,7 +162,7 @@ class HeadlessParserServiceTest {
//HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
MetricQueryReq metricCommand = new MetricQueryReq();
MetricQueryParam metricCommand = new MetricQueryParam();
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
@@ -174,7 +174,7 @@ class HeadlessParserServiceTest {
addDepartment(semanticSchema);
MetricQueryReq metricCommand2 = new MetricQueryReq();
MetricQueryParam metricCommand2 = new MetricQueryParam();
metricCommand2.setDimensions(new ArrayList<>(
Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));