topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-20 06:34:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless translator.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-24 19:07:56 +08:00
parent c22e3ef2e8
commit 860fd5d299
45 changed files with 795 additions and 1058 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java
View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import lombok.Data;
import java.util.List;
@Data
public class DataSetQueryParam {
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;
private boolean withAlias = true;
}

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java
View File

@@ -1,17 +0,0 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import lombok.Data;
import java.util.List;
@Data
public class MetricQueryParam {
private List<String> metrics;
private List<String> dimensions;
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,34 +1,25 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import java.util.List;
@Data
public class QueryStatement {
private Long dataSetId;
private List<Long> modelIds;
private String sql;
private String errMsg;
private QueryParam queryParam;
private MetricQueryParam metricQueryParam;
private DataSetQueryParam dataSetQueryParam;
private StructQueryParam structQueryParam;
private SqlQueryParam sqlQueryParam;
private OntologyQueryParam ontologyQueryParam;
private Integer status = 0;
private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges;
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;
private String dataSetSql;
private String dataSetAlias;
private String dataSetSimplifySql;
private Boolean enableLimitWrapper = false;
private Ontology ontology;
private SemanticSchemaResp semanticSchemaResp;
private Integer limit = 1000;
@@ -41,9 +32,4 @@ public class QueryStatement {
public boolean isTranslated() {
return isTranslated != null && isTranslated && isOk();
}
public QueryStatement error(String msg) {
this.setErrMsg(msg);
return this;
}
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.core.pojo;
import lombok.Data;
@Data
public class SqlQueryParam {
private String sql;
private String table;
private boolean supportWith = true;
private boolean withAlias = true;
private String simplifiedSql;
}

25
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Param;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class StructQueryParam {
private List<String> groups = new ArrayList();
private List<Aggregator> aggregators = new ArrayList();
private List<Order> orders = new ArrayList();
private List<Filter> dimensionFilters = new ArrayList();
private List<Filter> metricFilters = new ArrayList();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType;
private List<Param> params = new ArrayList<>();
}

524
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -1,507 +1,97 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
@Slf4j
public class DefaultSemanticTranslator implements SemanticTranslator {
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
public void translate(QueryStatement queryStatement) {
if (queryStatement.isTranslated()) {
return;
}
try {
preprocess(queryStatement);
parse(queryStatement);
optimize(queryStatement);
for (QueryConverter converter : ComponentFactory.getQueryConverters()) {
if (converter.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", converter.getClass().getName());
converter.convert(queryStatement);
}
}
doOntologyParse(queryStatement);
if (StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSimplifiedSql())) {
queryStatement.setSql(queryStatement.getSqlQueryParam().getSimplifiedSql());
}
if (StringUtils.isBlank(queryStatement.getSql())) {
throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
}
if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) {
queryStatement
.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit());
}
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
}
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("Failed to translate semantic query [{}]", e);
}
}
private void parse(QueryStatement queryStatement) throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getDataSetQueryParam())) {
queryStatement.setDataSetQueryParam(new DataSetQueryParam());
}
if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricQueryParam(new MetricQueryParam());
private void doOntologyParse(QueryStatement queryStatement) throws Exception {
OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam();
SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam();
log.info("parse with ontology: [{}]", ontologyQueryParam);
ComponentFactory.getQueryParser().parse(queryStatement);
if (!queryStatement.isOk()) {
throw new Exception(String.format("parse table [%s] error [%s]",
sqlQueryParam.getTable(), queryStatement.getErrMsg()));
}
log.debug("SemanticConverter before [{}]", queryParam);
for (QueryConverter headlessConverter : ComponentFactory.getQueryConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.debug("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(sqlQueryParam.getTable(), queryStatement.getSql()));
if (sqlQueryParam.isSupportWith()) {
EngineType engineType =
EngineType.fromString(queryStatement.getOntology().getDatabase().getType());
if (!SqlMergeWithUtils.hasWith(engineType, sqlQueryParam.getSql())) {
String withSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
.collect(Collectors.joining(",")) + "\n" + sqlQueryParam.getSql();
queryStatement.setSql(withSql);
} else {
List<String> parentTableList =
tables.stream().map(Pair::getLeft).collect(Collectors.toList());
List<String> parentSqlList =
tables.stream().map(Pair::getRight).collect(Collectors.toList());
String mergeSql = SqlMergeWithUtils.mergeWith(engineType, sqlQueryParam.getSql(),
parentSqlList, parentTableList);
queryStatement.setSql(mergeSql);
}
}
log.debug("SemanticConverter after {} {} {}", queryParam,
queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam());
if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) {
doParse(queryStatement.getDataSetQueryParam(), queryStatement);
} else {
queryStatement.getMetricQueryParam()
.setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
doParse(queryStatement,
AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
}
if (StringUtils.isEmpty(queryStatement.getSql())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
if (StringUtils.isNotBlank(queryStatement.getSql())
&& !SqlSelectHelper.hasLimit(queryStatement.getSql())) {
String querySql =
queryStatement.getSql() + " limit " + queryStatement.getLimit().toString();
queryStatement.setSql(querySql);
}
}
private QueryStatement doParse(DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
Ontology ontology = queryStatement.getOntology();
EngineType engineType = EngineType.fromString(ontology.getDatabase().getType());
try {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
boolean isSingleTable = dataSetQueryParam.getTables().size() == 1;
for (MetricTable metricTable : dataSetQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable,
dataSetQueryParam, queryStatement);
if (isSingleTable && StringUtils.isNotBlank(tableSql.getDataSetSimplifySql())) {
queryStatement.setSql(tableSql.getDataSetSimplifySql());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
tables.add(new String[] {metricTable.getAlias(), tableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql;
if (dataSetQueryParam.isSupportWith()) {
if (!SqlMergeWithUtils.hasWith(engineType, dataSetQueryParam.getSql())) {
sql = "with "
+ tables.stream()
.map(t -> String.format("%s as (%s)", t[0], t[1]))
.collect(Collectors.joining(","))
+ "\n" + dataSetQueryParam.getSql();
} else {
List<String> parentWithNameList = tables.stream().map(table -> table[0])
.collect(Collectors.toList());
List<String> parentSqlList = tables.stream().map(table -> table[1])
.collect(Collectors.toList());
sql = SqlMergeWithUtils.mergeWith(engineType,
dataSetQueryParam.getSql(), parentSqlList, parentWithNameList);
}
} else {
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ") "
+ (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
String dsSql = sqlQueryParam.getSql();
for (Pair<String, String> tb : tables) {
dsSql = StringUtils.replace(dsSql, tb.getLeft(), "(" + tb.getRight() + ") "
+ (sqlQueryParam.isWithAlias() ? "" : tb.getLeft()), -1);
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
private QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg);
try {
ComponentFactory.getQueryParser().parse(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception {
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(metricTable.getMetrics());
metricQueryParam.setDimensions(metricTable.getDimensions());
metricQueryParam.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricQueryParam.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricQueryParam(metricQueryParam);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setDataSetId(queryStatement.getDataSetId());
tableSql.setOntology(queryStatement.getOntology());
if (isSingleMetricTable) {
tableSql.setDataSetSql(dataSetQueryParam.getSql());
tableSql.setDataSetAlias(metricTable.getAlias());
}
tableSql = doParse(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]",
metricTable.getAlias(), tableSql.getErrMsg()));
}
return tableSql;
}
private void optimize(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
queryStatement.setSql(dsSql);
}
}
private void preprocess(QueryStatement queryStatement) {
if (StringUtils.isBlank(queryStatement.getSql())) {
return;
}
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
convertNameToBizName(queryStatement);
rewriteFunction(queryStatement);
queryStatement.setSql(SqlRemoveHelper.removeUnderscores(queryStatement.getSql()));
String tableName = SqlSelectHelper.getTableName(queryStatement.getSql());
if (StringUtils.isEmpty(tableName)) {
return;
}
// correct order item is same as agg alias
String reqSql = queryStatement.getSql();
queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(queryStatement.getSql()));
log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql());
// 5.build MetricTables
List<String> allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql());
List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
List<String> metrics =
metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList());
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
metricTable.getMetrics().addAll(metrics);
metricTable.getDimensions().addAll(dimensions);
metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, metricTable.getDimensions())));
} else {
queryStructReq.getAggregators()
.addAll(metricTable.getMetrics().stream()
.map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN))
.collect(Collectors.toList()));
}
AggOption aggOption = getAggOption(queryStatement, metricSchemas);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
// 6.build ParseSqlReq
DataSetQueryParam datasetQueryParam = new DataSetQueryParam();
datasetQueryParam.setTables(tables);
datasetQueryParam.setSql(queryStatement.getSql());
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
datasetQueryParam.setSupportWith(false);
datasetQueryParam.setWithAlias(false);
}
// 7. do deriveMetric
generateDerivedMetric(semanticSchemaResp, aggOption, datasetQueryParam);
// 8.physicalSql by ParseSqlReq
// queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryStatement.getSql()));
queryStructReq.setDataSetId(queryStatement.getDataSetId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
BeanUtils.copyProperties(queryStructReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setDataSetQueryParam(datasetQueryParam);
// queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
}
private AggOption getAggOption(QueryStatement queryStatement,
List<MetricSchemaResp> metricSchemas) {
String sql = queryStatement.getSql();
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to DEFAULT");
return AggOption.DEFAULT;
}
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.OUTER;
}
// if (queryStatement.isInnerLayerNative()) {
// return AggOption.NATIVE;
// }
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)
|| SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.OUTER;
}
long defaultAggNullCnt = metricSchemas.stream().filter(
m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg()))
.count();
if (defaultAggNullCnt > 0) {
log.debug("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.OUTER;
}
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = queryStatement.getSql();
log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceSqlByPositions(sql);
log.debug("replaceSqlByPositions:{}", sql);
sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + queryStatement.getDataSetId());
log.debug("replaceTableName after:{}", sql);
queryStatement.setSql(sql);
}
private Set<String> getDimensions(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toSet());
}
private List<MetricSchemaResp> getMetrics(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, MetricSchemaResp> metricLowerToNameMap =
semanticSchemaResp.getMetrics().stream().collect(Collectors
.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry));
return allFields.stream()
.filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toList());
}
private void rewriteFunction(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector =
engineAdaptor.functionNameCorrector(queryStatement.getSql());
queryStatement.setSql(functionNameCorrector);
}
}
protected Map<String, String> getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = semanticSchemaResp.getMetrics().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name,
String bizName) {
Set<Pair<String, String>> elements = new HashSet<>();
elements.add(Pair.of(name, bizName));
if (StringUtils.isNotBlank(aliasStr)) {
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
for (String alias : aliasList) {
elements.add(Pair.of(alias, bizName));
}
}
return elements.stream();
}
private QueryType getQueryType(AggOption aggOption) {
boolean isAgg = AggOption.isAgg(aggOption);
QueryType queryType = QueryType.DETAIL;
if (isAgg) {
queryType = QueryType.AGGREGATE;
}
return queryType;
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
DataSetQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) {
Set<String> measures = new HashSet<>();
Map<String, String> replaces = generateDerivedMetric(semanticSchemaResp, aggOption,
metricTable.getMetrics(), metricTable.getDimensions(), measures);
if (!CollectionUtils.isEmpty(replaces)) {
// metricTable sql use measures replace metric
sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces);
metricTable.setAggOption(AggOption.NATIVE);
// metricTable use measures replace metric
if (!CollectionUtils.isEmpty(measures)) {
metricTable.setMetrics(new ArrayList<>(measures));
} else {
// empty measure , fill default
metricTable.setMetrics(new ArrayList<>());
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, metricTable.getDimensions())));
}
}
}
viewQueryParam.setSql(sql);
}
private Map<String, String> generateDerivedMetric(SemanticSchemaResp semanticSchemaResp,
AggOption aggOption, List<String> metrics, List<String> dimensions,
Set<String> measures) {
Map<String, String> result = new HashMap<>();
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// Check if any metric is derived
boolean hasDerivedMetrics =
metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType
.isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams()));
if (!hasDerivedMetrics) {
return result;
}
log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
semanticSchemaResp.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getMeasures() != null) {
modelResp.getModelDetail().getMeasures()
.forEach(measure -> allMeasures.put(measure.getBizName(), measure));
}
});
Set<String> derivedDimensions = new HashSet<>();
Set<String> derivedMetrics = new HashSet<>();
Map<String, String> visitedMetrics = new HashMap<>();
for (MetricResp metricResp : metricResps) {
if (metrics.contains(metricResp.getBizName())) {
boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(),
metricResp.getMetricDefineByMeasureParams());
if (isDerived) {
String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields,
allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp),
metricResp.getMetricDefineType(), aggOption, visitedMetrics,
derivedMetrics, derivedDimensions);
result.put(metricResp.getBizName(), expr);
log.debug("derived metric {}->{}", metricResp.getBizName(), expr);
} else {
measures.add(metricResp.getBizName());
}
}
}
measures.addAll(derivedMetrics);
derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension))
.forEach(dimensions::add);
return result;
}
private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List<String> dimensions) {
if (!CollectionUtils.isEmpty(dimensions)) {
Map<String, Long> modelMatchCnt = new HashMap<>();
for (ModelResp modelResp : semanticSchemaResp.getModelResps()) {
modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions()
.stream().filter(d -> dimensions.contains(d.getBizName())).count());
}
return modelMatchCnt.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.map(m -> m.getKey()).findFirst().orElse("");
}
return semanticSchemaResp.getModelResps().get(0).getBizName();
}
}

30
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java
View File

@@ -1,14 +1,12 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/** Remove the default metric added by the system when the query only has dimensions */
@Slf4j
@@ -17,26 +15,26 @@ public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStatement queryStatement) {
QueryParam queryParam = queryStatement.getQueryParam();
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
String sqlRaw = queryStatement.getSql().trim();
if (StringUtils.isEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.debug("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryParam)) {
if (queryParam.getMetrics().size() == 0
&& !CollectionUtils.isEmpty(queryParam.getGroups())) {
String sqlForm = "select %s from ( %s ) src_no_metric";
String sql = String.format(sqlForm,
queryParam.getGroups().stream().collect(Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql);
}
}
// if (isDetailQuery(structQueryParam)) {
// if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) {
// String sqlForm = "select %s from ( %s ) src_no_metric";
// String sql = String.format(sqlForm,
// structQueryParam.getGroups().stream().collect(Collectors.joining(",")),
// sqlRaw);
// queryStatement.setSql(sql);
// }
// }
log.debug("after handleNoMetric, sql:{}", queryStatement.getSql());
}
public boolean isDetailQuery(QueryParam queryParam) {
return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryParam.getMetrics());
public boolean isDetailQuery(StructQueryParam structQueryParam) {
return Objects.nonNull(structQueryParam)
&& structQueryParam.getQueryType().isNativeAggQuery();
}
}

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java
View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/** A query parser generates physical SQL for the QueryStatement. */
public interface QueryParser {
void parse(QueryStatement queryStatement, AggOption aggOption) throws Exception;
void parse(QueryStatement queryStatement) throws Exception;
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.core.translator.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.QueryParser;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
@@ -16,7 +15,7 @@ import org.springframework.stereotype.Component;
public class CalciteQueryParser implements QueryParser {
@Override
public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception {
public void parse(QueryStatement queryStatement) throws Exception {
Ontology ontology = queryStatement.getOntology();
if (ontology == null) {
queryStatement.setErrMsg("No ontology could be found");
@@ -29,7 +28,7 @@ public class CalciteQueryParser implements QueryParser {
.enableOptimize(queryStatement.getEnableOptimize()).build())
.build();
SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema);
sqlBuilder.build(queryStatement, isAgg);
sqlBuilder.build(queryStatement);
}
}

19
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java Normal file
View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.headless.core.translator.calcite.s2sql;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import lombok.Data;
import java.util.List;
@Data
public class OntologyQueryParam {
private List<String> metrics = Lists.newArrayList();
private List<String> dimensions = Lists.newArrayList();
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
private AggOption aggOption = AggOption.DEFAULT;
}

62
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java
View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender;
@@ -17,23 +16,16 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.render.Render
import com.tencent.supersonic.headless.core.translator.calcite.sql.render.SourceRender;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.*;
/** parsing from query dimensions and metrics */
@Slf4j
public class SqlBuilder {
private final S2CalciteSchema schema;
private MetricQueryParam metricQueryParam;
private OntologyQueryParam ontologyQueryParam;
private SqlValidatorScope scope;
private SqlNode parserNode;
private boolean isAgg = false;
@@ -43,45 +35,32 @@ public class SqlBuilder {
this.schema = schema;
}
public void build(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricQueryParam = queryStatement.getMetricQueryParam();
if (metricQueryParam.getMetrics() == null) {
metricQueryParam.setMetrics(new ArrayList<>());
public void build(QueryStatement queryStatement) throws Exception {
this.ontologyQueryParam = queryStatement.getOntologyQueryParam();
if (ontologyQueryParam.getMetrics() == null) {
ontologyQueryParam.setMetrics(new ArrayList<>());
}
if (metricQueryParam.getDimensions() == null) {
metricQueryParam.setDimensions(new ArrayList<>());
if (ontologyQueryParam.getDimensions() == null) {
ontologyQueryParam.setDimensions(new ArrayList<>());
}
if (metricQueryParam.getLimit() == null) {
metricQueryParam.setLimit(0L);
if (ontologyQueryParam.getLimit() == null) {
ontologyQueryParam.setLimit(0L);
}
this.aggOption = aggOption;
this.aggOption = ontologyQueryParam.getAggOption();
buildParseNode();
Database database = queryStatement.getOntology().getDatabase();
EngineType engineType = EngineType.fromString(database.getType());
optimizeParseNode(engineType);
String sql = getSql(engineType);
queryStatement.setSql(sql);
if (Objects.nonNull(queryStatement.getEnableOptimize())
&& queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getDataSetAlias())
&& !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql
String simplifySql = rewrite(getSqlByDataSet(engineType, sql,
queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setDataSetSimplifySql(simplifySql);
}
}
}
private void buildParseNode() throws Exception {
// find the match Datasource
scope = SchemaBuilder.getScope(schema);
List<DataModel> dataModels =
DataModelNode.getRelatedDataModels(scope, schema, metricQueryParam);
DataModelNode.getRelatedDataModels(scope, schema, ontologyQueryParam);
if (dataModels == null || dataModels.isEmpty()) {
throw new Exception("data model not found");
}
@@ -98,14 +77,14 @@ public class SqlBuilder {
while (it.hasNext()) {
Renderer renderer = it.next();
if (previous != null) {
previous.render(metricQueryParam, dataModels, scope, schema, !isAgg);
previous.render(ontologyQueryParam, dataModels, scope, schema, !isAgg);
renderer.setTable(previous
.builderAs(DataModelNode.getNames(dataModels) + "_" + String.valueOf(i)));
i++;
}
previous = renderer;
}
builders.getLast().render(metricQueryParam, dataModels, scope, schema, !isAgg);
builders.getLast().render(ontologyQueryParam, dataModels, scope, schema, !isAgg);
parserNode = builders.getLast().builder();
}
@@ -116,7 +95,7 @@ public class SqlBuilder {
// default by dataModel time aggregation
if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime()
.equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (!metricQueryParam.isNativeQuery()) {
if (!ontologyQueryParam.isNativeQuery()) {
return true;
}
}
@@ -164,13 +143,4 @@ public class SqlBuilder {
}
}
private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql,
String parentAlias) throws SqlParseException {
if (!SqlMergeWithUtils.hasWith(engineType, dataSetSql)) {
return String.format("with %s as (%s) %s", parentAlias, parentSql, dataSetSql);
}
return SqlMergeWithUtils.mergeWith(engineType, dataSetSql,
Collections.singletonList(parentSql), Collections.singletonList(parentAlias));
}
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java
View File

@@ -4,13 +4,13 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.SchemaBuilder;
import lombok.extern.slf4j.Slf4j;
@@ -150,7 +150,7 @@ public class DataModelNode extends SemanticNode {
}
public static void getQueryDimensionMeasure(S2CalciteSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY)
? d.split(Constants.DIMENSION_IDENTIFY)[1]
@@ -166,7 +166,7 @@ public class DataModelNode extends SemanticNode {
}
public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
@@ -192,7 +192,7 @@ public class DataModelNode extends SemanticNode {
}
public static List<DataModel> getRelatedDataModels(SqlValidatorScope scope,
S2CalciteSchema schema, MetricQueryParam metricCommand) throws Exception {
S2CalciteSchema schema, OntologyQueryParam metricCommand) throws Exception {
List<DataModel> dataModels = new ArrayList<>();
// check by metric
@@ -208,7 +208,7 @@ public class DataModelNode extends SemanticNode {
sourceMeasure.retainAll(measures);
dataSourceMeasures.put(entry.getKey(), sourceMeasure.size());
}
log.info("dataSourceMeasures [{}]", dataSourceMeasures);
log.info("metrics: [{}]", dataSourceMeasures);
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) {
@@ -267,7 +267,7 @@ public class DataModelNode extends SemanticNode {
}
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension,
List<String> measures, Set<String> dimension, MetricQueryParam metricCommand,
List<String> measures, Set<String> dimension, OntologyQueryParam metricCommand,
SqlValidatorScope scope, EngineType engineType) throws Exception {
boolean isAllMatch = true;
sourceMeasure.retainAll(measures);

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode;
@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
public class FilterRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView;
SqlNode filterNode = null;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
@@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode;
@@ -47,7 +47,7 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode;
@@ -22,7 +22,7 @@ import java.util.List;
public class OutputRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView;
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode;
@@ -114,6 +114,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build());
}
public abstract void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public abstract void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception;
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
@@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode;
@@ -336,9 +336,9 @@ public class SourceRender extends Renderer {
}
}
public void render(MetricQueryParam metricQueryParam, List<DataModel> dataModels,
public void render(OntologyQueryParam ontologyQueryParam, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryParam.getWhere();
String queryWhere = ontologyQueryParam.getWhere();
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
@@ -349,13 +349,13 @@ public class SourceRender extends Renderer {
}
if (dataModels.size() == 1) {
DataModel dataModel = dataModels.get(0);
super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataModel, scope,
schema, nonAgg);
super.tableView = renderOne("", fieldWhere, ontologyQueryParam.getMetrics(),
ontologyQueryParam.getDimensions(), ontologyQueryParam.getWhere(), dataModel,
scope, schema, nonAgg);
return;
}
JoinRender joinRender = new JoinRender();
joinRender.render(metricQueryParam, dataModels, scope, schema, nonAgg);
joinRender.render(ontologyQueryParam, dataModels, scope, schema, nonAgg);
super.tableView = joinRender.getTableView();
}
}

15
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java
View File

@@ -4,7 +4,6 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import lombok.extern.slf4j.Slf4j;
@@ -28,8 +27,8 @@ public class DefaultDimValueConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
return !Objects.isNull(queryStatement.getDataSetQueryParam())
&& !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql());
return Objects.nonNull(queryStatement.getSqlQueryParam())
&& StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSql());
}
@Override
@@ -40,15 +39,13 @@ public class DefaultDimValueConverter implements QueryConverter {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
String sql = queryStatement.getDataSetQueryParam().getSql();
String sql = queryStatement.getSqlQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}
MetricTable metricTable =
queryStatement.getDataSetQueryParam().getTables().stream().findFirst().orElse(null);
List<Expression> expressions = Lists.newArrayList();
for (Dimension dimension : dimensions) {
ExpressionList expressionList = new ExpressionList();
@@ -59,11 +56,11 @@ public class DefaultDimValueConverter implements QueryConverter {
inExpression.setLeftExpression(new Column(dimension.getBizName()));
inExpression.setRightExpression(expressionList);
expressions.add(inExpression);
if (metricTable != null) {
metricTable.getDimensions().add(dimension.getBizName());
if (Objects.nonNull(queryStatement.getSqlQueryParam().getTable())) {
queryStatement.getOntologyQueryParam().getDimensions().add(dimension.getBizName());
}
}
sql = SqlAddHelper.addWhere(sql, expressions);
queryStatement.getDataSetQueryParam().setSql(sql);
queryStatement.getSqlQueryParam().setSql(sql);
}
}

291
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java
View File

@@ -6,82 +6,46 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/** supplement the QueryStatement when query with custom aggregation method */
@Component("CalculateAggConverter")
@Slf4j
public class CalculateAggConverter implements QueryConverter {
public class MetricRatioConverter implements QueryConverter {
public interface EngineSql {
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam),
sqlGenerateUtils.getOrderBy(queryParam), sqlGenerateUtils.getLimit(queryParam));
}
sqlCommand.setSql(sql);
return sqlCommand;
String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql);
}
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
if (Objects.isNull(queryStatement.getStructQueryParam()) || queryStatement.getIsS2SQL()
|| !isRatioAccept(queryStatement.getStructQueryParam())) {
return false;
}
QueryParam queryParam = queryStatement.getQueryParam();
if (queryParam.getQueryType().isNativeAggQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryParam.getAggregators())) {
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
if (structQueryParam.getQueryType().isNativeAggQuery()
|| CollectionUtils.isEmpty(structQueryParam.getAggregators())) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryParam.getAggregators()) {
for (Aggregator agg : structQueryParam.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
@@ -98,14 +62,13 @@ public class CalculateAggConverter implements QueryConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
Database database = queryStatement.getOntology().getDatabase();
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
generateRatioSql(queryStatement, EngineType.fromString(database.getType().toUpperCase()),
database.getVersion());
}
/** Ratio */
public boolean isRatioAccept(QueryParam queryParam) {
Long ratioFuncNum = queryParam.getAggregators().stream()
public boolean isRatioAccept(StructQueryParam structQueryParam) {
Long ratioFuncNum = structQueryParam.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)
|| f.getFunc().equals(AggOperatorEnum.RATIO_OVER)))
.count();
@@ -115,53 +78,47 @@ public class CalculateAggConverter implements QueryConverter {
return false;
}
public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
public void generateRatioSql(QueryStatement queryStatement, EngineType engineTypeEnum,
String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
check(structQueryParam);
queryStatement.setEnableOptimize(false);
DataSetQueryParam sqlCommand = new DataSetQueryParam();
OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam();
ontologyQueryParam.setAggOption(AggOption.AGGREGATION);
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryParam);
boolean isOver = isOverRatio(structQueryParam);
String sql = "";
SqlQueryParam dsParam = queryStatement.getSqlQueryParam();
dsParam.setTable(metricTableName);
switch (engineTypeEnum) {
case H2:
sql = new H2EngineSql().sql(queryParam, isOver, true, metricTableName);
sql = new H2EngineSql().sql(structQueryParam, isOver, true, metricTableName);
break;
case MYSQL:
case DORIS:
case CLICKHOUSE:
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
dsParam.setSupportWith(false);
}
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
metricTableName);
sql = new MysqlEngineSql().sql(structQueryParam, isOver,
dsParam.isSupportWith(), metricTableName);
} else {
sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
sql = new CkEngineSql().sql(structQueryParam, isOver, dsParam.isSupportWith(),
metricTableName);
}
break;
default:
}
sqlCommand.setSql(sql);
return sqlCommand;
dsParam.setSql(sql);
}
public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) {
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
@@ -171,43 +128,44 @@ public class CalculateAggConverter implements QueryConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver,
boolean isAdd) {
if (Objects.nonNull(structQueryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
return "day," + (isOver ? addStr + "7" : addStr + "1");
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return isOver ? "month," + addStr + "1" : "day," + addStr + "7";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) {
return isOver ? "year," + addStr + "1" : "month," + addStr + "1";
}
}
return "";
}
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan,
aliasRight + timeDim);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim,
getTimeSpan(structQueryParam, isOver, false), aliasLeft + timeDim,
aliasRight + timeDim);
}
return String.format("%s = TIMESTAMPADD(%s,%s) ", aliasLeft + timeDim, timeSpan,
@@ -217,7 +175,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -228,35 +186,36 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
return sql;
}
}
public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
@@ -266,7 +225,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -277,45 +236,49 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
if (!asWith) {
return String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver),
getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."),
getOrderBy(structQueryParam), getLimit(structQueryParam));
}
return String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."), getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
metricSql, metricSql, getOverSelect(structQueryParam, isOver),
getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."),
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
}
}
public class MysqlEngineSql implements EngineSql {
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver,
boolean isAdd) {
if (Objects.nonNull(structQueryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
return isOver ? addStr + "7 day" : addStr + "1 day";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) {
return isOver ? addStr + "1 month" : addStr + "7 day";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return isOver ? addStr + "1 year" : addStr + "1 month";
}
}
return "";
}
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) {
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
@@ -325,26 +288,26 @@ public class CalculateAggConverter implements QueryConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
@@ -354,7 +317,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -365,51 +328,53 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
return sql;
}
}
private String getAllJoinSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream()
private String getAllJoinSelect(StructQueryParam structQueryParam, String alias) {
String aggStr = structQueryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") + "_roll")
.collect(Collectors.joining(","));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
groups.add(alias + group + " as " + group + "_roll");
}
return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr;
}
private String getGroupDimWithOutTime(QueryParam queryParam) {
String timeDim = getTimeDim(queryParam);
return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
private String getGroupDimWithOutTime(StructQueryParam structQueryParam) {
String timeDim = getTimeDim(structQueryParam);
return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(QueryParam queryParam) {
private static String getTimeDim(StructQueryParam structQueryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(queryParam.getDateInfo());
return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo());
}
private static String getLimit(QueryParam queryParam) {
if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) {
return " limit " + String.valueOf(queryParam.getLimit());
private static String getLimit(StructQueryParam structQueryParam) {
if (structQueryParam != null && structQueryParam.getLimit() != null
&& structQueryParam.getLimit() > 0) {
return " limit " + String.valueOf(structQueryParam.getLimit());
}
return "";
}
private String getAllSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr;
private String getAllSelect(StructQueryParam structQueryParam, String alias) {
String aggStr = structQueryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias)).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, structQueryParam.getGroups()) + "," + aggStr;
}
private String getSelectField(final Aggregator agg, String alias) {
@@ -421,32 +386,32 @@ public class CalculateAggConverter implements QueryConverter {
return sqlGenerateUtils.getSelectField(agg);
}
private String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
private String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryParam.getGroups());
return "group by " + String.join(",", structQueryParam.getGroups());
}
private static String getOrderBy(QueryParam queryParam) {
return "order by " + getTimeDim(queryParam) + " desc";
private static String getOrderBy(StructQueryParam structQueryParam) {
return "order by " + getTimeDim(structQueryParam) + " desc";
}
private boolean isOverRatio(QueryParam queryParam) {
Long overCt = queryParam.getAggregators().stream()
private boolean isOverRatio(StructQueryParam structQueryParam) {
Long overCt = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0;
}
private void check(QueryParam queryParam) throws Exception {
Long ratioOverNum = queryParam.getAggregators().stream()
private void check(StructQueryParam structQueryParam) throws Exception {
Long ratioOverNum = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryParam.getAggregators().stream()
Long ratioRollNum = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}
if (getTimeDim(queryParam).isEmpty()) {
if (getTimeDim(structQueryParam).isEmpty()) {
throw new Exception("miss time filter");
}
}

74
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java
View File

@@ -1,74 +0,0 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/** QueryConverter default implement */
@Component("ParserDefaultConverter")
@Slf4j
public class ParserDefaultConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false;
}
CalculateAggConverter calculateConverterAgg =
ContextUtils.getBean(CalculateAggConverter.class);
return !calculateConverterAgg.accept(queryStatement);
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
MetricQueryParam metricReq =
generateSqlCommand(queryStatement.getQueryParam(), queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null));
BeanUtils.copyProperties(metricReq, metricQueryParam);
}
public MetricQueryParam generateSqlCommand(QueryParam queryParam,
QueryStatement queryStatement) {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(queryParam.getMetrics());
metricQueryParam.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryParam.setWhere(where);
metricQueryParam.setOrder(queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
metricQueryParam.setLimit(queryParam.getLimit());
// support detail query
if (queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataModel> modelMap = queryStatement.getOntology().getModelMap();
for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName();
String internalMetricName =
sqlGenerateUtils.generateInternalMetricName(modelBizName);
metricQueryParam.getMetrics().add(internalMetricName);
}
}
return metricQueryParam;
}
}

308
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java Normal file
View File

@@ -0,0 +1,308 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component("SqlQueryConverter")
@Slf4j
public class SqlQueryConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.nonNull(queryStatement.getSqlQueryParam()) && queryStatement.getIsS2SQL()) {
return true;
}
return false;
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
convertNameToBizName(queryStatement);
rewriteFunction(queryStatement);
String reqSql = queryStatement.getSqlQueryParam().getSql();
String tableName = SqlSelectHelper.getTableName(reqSql);
if (StringUtils.isEmpty(tableName)) {
return;
}
// replace order by field with the select sequence number
queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderbyField(reqSql));
log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql());
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
// fill dataSetQuery
SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam();
sqlQueryParam.setTable(tableName.toLowerCase());
if (!sqlGenerateUtils.isSupportWith(
EngineType.fromString(semanticSchemaResp.getDatabaseResp().getType().toUpperCase()),
semanticSchemaResp.getDatabaseResp().getVersion())) {
sqlQueryParam.setSupportWith(false);
sqlQueryParam.setWithAlias(false);
}
// build ontologyQuery
List<String> allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql());
List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
List<String> metrics =
metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList());
AggOption aggOption = getAggOption(queryStatement, metricSchemas);
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
OntologyQueryParam ontologyQueryParam = new OntologyQueryParam();
ontologyQueryParam.getMetrics().addAll(metrics);
ontologyQueryParam.getDimensions().addAll(dimensions);
ontologyQueryParam.setAggOption(aggOption);
ontologyQueryParam.setNativeQuery(!AggOption.isAgg(aggOption));
log.info("parse sqlQuery [{}] ", sqlQueryParam);
queryStatement.setOntologyQueryParam(ontologyQueryParam);
queryStatement.setSql(sqlQueryParam.getSql());
generateDerivedMetric(sqlGenerateUtils, queryStatement);
}
private AggOption getAggOption(QueryStatement queryStatement,
List<MetricSchemaResp> metricSchemas) {
String sql = queryStatement.getSql();
if (SqlSelectFunctionHelper.hasAggregateFunction(sql)) {
return AggOption.AGGREGATION;
}
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to DEFAULT");
return AggOption.DEFAULT;
}
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.OUTER;
}
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)
|| SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.OUTER;
}
long defaultAggNullCnt = metricSchemas.stream().filter(
m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg()))
.count();
if (defaultAggNullCnt > 0) {
log.debug("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.OUTER;
}
return AggOption.DEFAULT;
}
private Set<String> getDimensions(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toSet());
}
private List<MetricSchemaResp> getMetrics(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, MetricSchemaResp> metricLowerToNameMap =
semanticSchemaResp.getMetrics().stream().collect(Collectors
.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry));
return allFields.stream()
.filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toList());
}
private void generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils,
QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
SqlQueryParam dsParam = queryStatement.getSqlQueryParam();
OntologyQueryParam ontology = queryStatement.getOntologyQueryParam();
String sql = dsParam.getSql();
Set<String> measures = new HashSet<>();
Map<String, String> replaces = generateDerivedMetric(sqlGenerateUtils, semanticSchemaResp,
ontology.getAggOption(), ontology.getMetrics(), ontology.getDimensions(), measures);
if (!CollectionUtils.isEmpty(replaces)) {
// metricTable sql use measures replace metric
sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces);
ontology.setAggOption(AggOption.NATIVE);
// metricTable use measures replace metric
if (!CollectionUtils.isEmpty(measures)) {
ontology.getMetrics().addAll(measures);
} else {
// empty measure , fill default
ontology.setMetrics(new ArrayList<>());
ontology.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, ontology.getDimensions())));
}
}
dsParam.setSql(sql);
}
private Map<String, String> generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils,
SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List<String> metrics,
List<String> dimensions, Set<String> measures) {
Map<String, String> result = new HashMap<>();
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// Check if any metric is derived
boolean hasDerivedMetrics =
metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType
.isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams()));
if (!hasDerivedMetrics) {
return result;
}
log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
semanticSchemaResp.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getMeasures() != null) {
modelResp.getModelDetail().getMeasures()
.forEach(measure -> allMeasures.put(measure.getBizName(), measure));
}
});
Set<String> derivedDimensions = new HashSet<>();
Set<String> derivedMetrics = new HashSet<>();
Map<String, String> visitedMetrics = new HashMap<>();
for (MetricResp metricResp : metricResps) {
if (metrics.contains(metricResp.getBizName())) {
boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(),
metricResp.getMetricDefineByMeasureParams());
if (isDerived) {
String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields,
allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp),
metricResp.getMetricDefineType(), aggOption, visitedMetrics,
derivedMetrics, derivedDimensions);
result.put(metricResp.getBizName(), expr);
log.debug("derived metric {}->{}", metricResp.getBizName(), expr);
} else {
measures.add(metricResp.getBizName());
}
}
}
measures.addAll(derivedMetrics);
derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension))
.forEach(dimensions::add);
return result;
}
private void convertNameToBizName(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = queryStatement.getSqlQueryParam().getSql();
log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + queryStatement.getDataSetId());
log.debug("replaceTableName after:{}", sql);
queryStatement.getSqlQueryParam().setSql(sql);
}
private void rewriteFunction(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
String sql = queryStatement.getSqlQueryParam().getSql();
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector = engineAdaptor.functionNameCorrector(sql);
queryStatement.getSqlQueryParam().setSql(functionNameCorrector);
}
}
protected Map<String, String> getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = semanticSchemaResp.getMetrics().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name,
String bizName) {
Set<Pair<String, String>> elements = new HashSet<>();
elements.add(Pair.of(name, bizName));
if (StringUtils.isNotBlank(aliasStr)) {
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
for (String alias : aliasList) {
elements.add(Pair.of(alias, bizName));
}
}
return elements.stream();
}
private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List<String> dimensions) {
if (!CollectionUtils.isEmpty(dimensions)) {
Map<String, Long> modelMatchCnt = new HashMap<>();
for (ModelResp modelResp : semanticSchemaResp.getModelResps()) {
modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions()
.stream().filter(d -> dimensions.contains(d.getBizName())).count());
}
return modelMatchCnt.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.map(m -> m.getKey()).findFirst().orElse("");
}
return semanticSchemaResp.getModelResps().get(0).getBizName();
}
}

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java
View File

@@ -14,12 +14,12 @@ import java.util.List;
import java.util.Objects;
@Slf4j
@Component("SqlVariableParseConverter")
public class SqlVariableParseConverter implements QueryConverter {
@Component("SqlVariableConverter")
public class SqlVariableConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam())) {
if (Objects.isNull(queryStatement.getStructQueryParam()) && queryStatement.getIsS2SQL()) {
return false;
}
return true;
@@ -38,7 +38,7 @@ public class SqlVariableParseConverter implements QueryConverter {
String sqlParsed =
SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(),
modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams());
queryStatement.getStructQueryParam().getParams());
DataModel dataModel =
queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName());
dataModel.setSqlQuery(sqlParsed);

74
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java Normal file
View File

@@ -0,0 +1,74 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.Objects;
import java.util.stream.Collectors;
@Component("ParserDefaultConverter")
@Slf4j
public class StructQueryConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.nonNull(queryStatement.getStructQueryParam()) && !queryStatement.getIsS2SQL()) {
return true;
}
return false;
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
String dsTable = "t_1";
SqlQueryParam sqlParam = new SqlQueryParam();
sqlParam.setTable(dsTable);
String sql = String.format("select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(structQueryParam), dsTable,
sqlGenerateUtils.getGroupBy(structQueryParam),
sqlGenerateUtils.getOrderBy(structQueryParam),
sqlGenerateUtils.getLimit(structQueryParam));
Database database = queryStatement.getOntology().getDatabase();
EngineType engineType = EngineType.fromString(database.getType().toUpperCase());
if (!sqlGenerateUtils.isSupportWith(engineType, database.getVersion())) {
sqlParam.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(structQueryParam), dsTable,
sqlGenerateUtils.getGroupBy(structQueryParam),
sqlGenerateUtils.getOrderBy(structQueryParam),
sqlGenerateUtils.getLimit(structQueryParam));
}
sqlParam.setSql(sql);
queryStatement.setSqlQueryParam(sqlParam);
OntologyQueryParam ontologyQueryParam = new OntologyQueryParam();
ontologyQueryParam.getDimensions().addAll(structQueryParam.getGroups());
ontologyQueryParam.getMetrics().addAll(structQueryParam.getAggregators().stream()
.map(a -> a.getColumn()).collect(Collectors.toList()));
String where = sqlGenerateUtils.generateWhere(structQueryParam, null);
ontologyQueryParam.setWhere(where);
ontologyQueryParam.setAggOption(AggOption.AGGREGATION);
ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery());
ontologyQueryParam.setOrder(structQueryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
ontologyQueryParam.setLimit(structQueryParam.getLimit());
queryStatement.setOntologyQueryParam(ontologyQueryParam);
log.info("parse structQuery [{}] ", queryStatement.getSqlQueryParam());
}
}

67
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -12,7 +12,6 @@ import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -20,6 +19,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.core.config.ExecutorConfig;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -85,25 +85,26 @@ public class SqlGenerateUtils {
return selectSql;
}
public String getLimit(QueryParam queryParam) {
if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) {
return " limit " + queryParam.getLimit();
public String getLimit(StructQueryParam structQueryParam) {
if (structQueryParam != null && structQueryParam.getLimit() != null
&& structQueryParam.getLimit() > 0) {
return " limit " + structQueryParam.getLimit();
}
return "";
}
public String getSelect(QueryParam queryParam) {
String aggStr = queryParam.getAggregators().stream().map(this::getSelectField)
public String getSelect(StructQueryParam structQueryParam) {
String aggStr = structQueryParam.getAggregators().stream().map(this::getSelectField)
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getSelect(QueryParam queryParam, Map<String, String> deriveMetrics) {
String aggStr = queryParam.getAggregators().stream()
public String getSelect(StructQueryParam structQueryParam, Map<String, String> deriveMetrics) {
String aggStr = structQueryParam.getAggregators().stream()
.map(a -> getSelectField(a, deriveMetrics)).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getSelectField(final Aggregator agg) {
@@ -128,46 +129,46 @@ public class SqlGenerateUtils {
return deriveMetrics.get(agg.getColumn());
}
public String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
public String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryParam.getGroups());
return "group by " + String.join(",", structQueryParam.getGroups());
}
public String getOrderBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
public String getOrderBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getOrders())) {
return "";
}
return "order by " + queryParam.getOrders().stream()
return "order by " + structQueryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String getOrderBy(QueryParam queryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
public String getOrderBy(StructQueryParam structQueryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(structQueryParam.getOrders())) {
return "";
}
if (!queryParam.getOrders().stream()
if (!structQueryParam.getOrders().stream()
.anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryParam);
return getOrderBy(structQueryParam);
}
return "order by " + queryParam.getOrders().stream()
return "order by " + structQueryParam.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn())
? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) {
public String generateWhere(StructQueryParam structQueryParam, ItemDateResp itemDateResp) {
String whereClauseFromFilter =
sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate);
sqlFilterUtils.getWhereClause(structQueryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(structQueryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(structQueryParam, whereClauseFromFilter, whereFromDate);
}
private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
String whereFromDate) {
private String mergeDateWhereClause(StructQueryParam structQueryParam,
String whereClauseFromFilter, String whereFromDate) {
if (StringUtils.isNotEmpty(whereFromDate)
&& StringUtils.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
@@ -179,7 +180,7 @@ public class SqlGenerateUtils {
return whereFromDate;
} else if (Objects.isNull(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) {
log.debug("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryParam.getDateInfo());
return dateModeUtils.defaultRecentDateInfo(structQueryParam.getDateInfo());
}
return whereClauseFromFilter;
}
@@ -203,12 +204,12 @@ public class SqlGenerateUtils {
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public Triple<String, String, String> getBeginEndTime(QueryParam queryParam,
public Triple<String, String, String> getBeginEndTime(StructQueryParam structQueryParam,
ItemDateResp dataDate) {
if (Objects.isNull(queryParam.getDateInfo())) {
if (Objects.isNull(structQueryParam.getDateInfo())) {
return Triple.of("", "", "");
}
DateConf dateConf = queryParam.getDateInfo();
DateConf dateConf = structQueryParam.getDateInfo();
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
if (dateInfo.isEmpty()) {
return Triple.of("", "", "");

3
headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.chat.core.parser.aggregate;
import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser;
import org.junit.jupiter.api.Test;
@@ -318,7 +317,7 @@ public class CalciteSqlParserTest {
+ " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}";
QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class);
CalciteQueryParser calciteSqlParser = new CalciteQueryParser();
calciteSqlParser.parse(queryStatement, AggOption.DEFAULT);
calciteSqlParser.parse(queryStatement);
Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""),
"SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`"
+ "GROUPBY`imp_date`,`imp_date`");