topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 13:07:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless core.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-17 22:54:45 +08:00
parent a0f53359ef
commit cd889b479c
24 changed files with 264 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -42,7 +42,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
} }
} }
public void parse(QueryStatement queryStatement) throws Exception { private void parse(QueryStatement queryStatement) throws Exception {
QueryParam queryParam = queryStatement.getQueryParam(); QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getDataSetQueryParam())) { if (Objects.isNull(queryStatement.getDataSetQueryParam())) {
queryStatement.setDataSetQueryParam(new DataSetQueryParam()); queryStatement.setDataSetQueryParam(new DataSetQueryParam());
@@ -64,7 +64,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
} else { } else {
queryStatement.getMetricQueryParam() queryStatement.getMetricQueryParam()
.setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); .setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
doParse(queryStatement); doParse(queryStatement,
AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
} }
if (StringUtils.isEmpty(queryStatement.getSql())) { if (StringUtils.isEmpty(queryStatement.getSql())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg()); throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
@@ -77,7 +78,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
} }
} }
public QueryStatement doParse(DataSetQueryParam dataSetQueryParam, private QueryStatement doParse(DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) { QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam); log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
SemanticModel semanticModel = queryStatement.getSemanticModel(); SemanticModel semanticModel = queryStatement.getSemanticModel();
@@ -132,12 +133,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
return queryStatement; return queryStatement;
} }
public QueryStatement doParse(QueryStatement queryStatement) { private QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) {
return doParse(queryStatement,
AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
}
public QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg); log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg);
try { try {

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.translator.QueryParser;
import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions; import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParseException;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -31,18 +31,18 @@ public class CalciteQueryParser implements QueryParser {
return; return;
} }
queryStatement.setMetricQueryParam(metricReq); queryStatement.setMetricQueryParam(metricReq);
SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); S2SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement);
AggPlanner aggBuilder = new AggPlanner(semanticSchema); AggPlanner aggPlanner = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, isAgg); aggPlanner.plan(queryStatement, isAgg);
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType()); EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType());
queryStatement.setSql(aggBuilder.getSql(engineType)); queryStatement.setSql(aggPlanner.getSql(engineType));
if (Objects.nonNull(queryStatement.getEnableOptimize()) if (Objects.nonNull(queryStatement.getEnableOptimize())
&& queryStatement.getEnableOptimize() && queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getDataSetAlias()) && Objects.nonNull(queryStatement.getDataSetAlias())
&& !queryStatement.getDataSetAlias().isEmpty()) { && !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql // simplify model sql with query sql
String simplifySql = aggBuilder.simplify( String simplifySql = aggPlanner.simplify(
getSqlByDataSet(engineType, aggBuilder.getSql(engineType), getSqlByDataSet(engineType, aggPlanner.getSql(engineType),
queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()),
engineType); engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
@@ -52,10 +52,10 @@ public class CalciteQueryParser implements QueryParser {
} }
} }
private SemanticSchema getSemanticSchema(SemanticModel semanticModel, private S2SemanticSchema getSemanticSchema(SemanticModel semanticModel,
QueryStatement queryStatement) { QueryStatement queryStatement) {
SemanticSchema semanticSchema = S2SemanticSchema semanticSchema =
SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); S2SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build();
semanticSchema.setSemanticModel(semanticModel); semanticSchema.setSemanticModel(semanticModel);
semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDatasource(semanticModel.getDatasourceMap());
semanticSchema.setDimension(semanticModel.getDimensionMap()); semanticSchema.setDimension(semanticModel.getDimensionMap());

64
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java
View File

@@ -7,11 +7,10 @@ import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender;
@@ -27,29 +26,27 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.Objects; import java.util.Objects;
import java.util.Stack;
/** parsing from query dimensions and metrics */ /** parsing from query dimensions and metrics */
@Slf4j @Slf4j
public class AggPlanner implements Planner { public class AggPlanner implements Planner {
private MetricQueryParam metricReq; private MetricQueryParam metricReq;
private SemanticSchema schema; private final S2SemanticSchema schema;
private SqlValidatorScope scope; private SqlValidatorScope scope;
private Stack<TableView> dataSets = new Stack<>();
private SqlNode parserNode; private SqlNode parserNode;
private String sourceId; private String sourceId;
private boolean isAgg = false; private boolean isAgg = false;
private AggOption aggOption = AggOption.DEFAULT; private AggOption aggOption = AggOption.DEFAULT;
public AggPlanner(SemanticSchema schema) { public AggPlanner(S2SemanticSchema schema) {
this.schema = schema; this.schema = schema;
} }
public void parse() throws Exception { private void parse() throws Exception {
// find the match Datasource // find the match Datasource
scope = SchemaBuilder.getScope(schema); scope = SchemaBuilder.getScope(schema);
List<DataSource> datasource = getMatchDataSource(scope); List<DataModel> datasource = getMatchDataSource(scope);
if (datasource == null || datasource.isEmpty()) { if (datasource == null || datasource.isEmpty()) {
throw new Exception("datasource not found"); throw new Exception("datasource not found");
} }
@@ -78,16 +75,16 @@ public class AggPlanner implements Planner {
parserNode = builders.getLast().builder(); parserNode = builders.getLast().builder();
} }
private List<DataSource> getMatchDataSource(SqlValidatorScope scope) throws Exception { private List<DataModel> getMatchDataSource(SqlValidatorScope scope) throws Exception {
return DataSourceNode.getMatchDataSources(scope, schema, metricReq); return DataSourceNode.getMatchDataSources(scope, schema, metricReq);
} }
private boolean getAgg(DataSource dataSource) { private boolean getAgg(DataModel dataModel) {
if (!AggOption.DEFAULT.equals(aggOption)) { if (!AggOption.DEFAULT.equals(aggOption)) {
return AggOption.isAgg(aggOption); return AggOption.isAgg(aggOption);
} }
// default by dataSource time aggregation // default by dataModel time aggregation
if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime() if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime()
.equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { .equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (!metricReq.isNativeQuery()) { if (!metricReq.isNativeQuery()) {
return true; return true;
@@ -97,7 +94,7 @@ public class AggPlanner implements Planner {
} }
@Override @Override
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception { public void plan(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricReq = queryStatement.getMetricQueryParam(); this.metricReq = queryStatement.getMetricQueryParam();
if (metricReq.getMetrics() == null) { if (metricReq.getMetrics() == null) {
metricReq.setMetrics(new ArrayList<>()); metricReq.setMetrics(new ArrayList<>());
@@ -129,22 +126,6 @@ public class AggPlanner implements Planner {
@Override @Override
public String simplify(String sql, EngineType engineType) { public String simplify(String sql, EngineType engineType) {
return optimize(sql, engineType);
}
public void optimize(EngineType engineType) {
if (Objects.isNull(schema.getRuntimeOptions())
|| Objects.isNull(schema.getRuntimeOptions().getEnableOptimize())
|| !schema.getRuntimeOptions().getEnableOptimize()) {
return;
}
SqlNode optimizeNode = optimizeSql(SemanticNode.getSql(parserNode, engineType), engineType);
if (Objects.nonNull(optimizeNode)) {
parserNode = optimizeNode;
}
}
public String optimize(String sql, EngineType engineType) {
try { try {
SqlNode sqlNode = SqlNode sqlNode =
SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt();
@@ -153,21 +134,32 @@ public class AggPlanner implements Planner {
SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType); SemanticNode.optimize(scope, schema, sqlNode, engineType), engineType);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("optimize error {}", e); log.error("optimize error {}", e.toString());
} }
return ""; return "";
} }
private SqlNode optimizeSql(String sql, EngineType engineType) { private void optimize(EngineType engineType) {
if (Objects.isNull(schema.getRuntimeOptions())
|| Objects.isNull(schema.getRuntimeOptions().getEnableOptimize())
|| !schema.getRuntimeOptions().getEnableOptimize()) {
return;
}
SqlNode optimizeNode = null;
try { try {
SqlNode sqlNode = SqlNode sqlNode = SqlParser.create(SemanticNode.getSql(parserNode, engineType),
SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt(); Configuration.getParserConfig(engineType)).parseStmt();
if (Objects.nonNull(sqlNode)) { if (Objects.nonNull(sqlNode)) {
return SemanticNode.optimize(scope, schema, sqlNode, engineType); optimizeNode = SemanticNode.optimize(scope, schema, sqlNode, engineType);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("optimize error {}", e); log.error("optimize error {}", e);
} }
return null;
if (Objects.nonNull(optimizeNode)) {
parserNode = optimizeNode;
}
} }
} }

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java
View File

@@ -7,11 +7,11 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/** parse and generate SQL and other execute information */ /** parse and generate SQL and other execute information */
public interface Planner { public interface Planner {
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception; void plan(QueryStatement queryStatement, AggOption aggOption) throws Exception;
public String getSql(EngineType enginType); String getSql(EngineType enginType);
public String getSourceId(); String getSourceId();
public String simplify(String sql, EngineType engineType); String simplify(String sql, EngineType engineType);
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataSource.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/DataModel.java
View File

@@ -7,7 +7,7 @@ import java.util.List;
@Data @Data
@Builder @Builder
public class DataSource { public class DataModel {
private Long id; private Long id;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/SemanticModel.java
View File

@@ -15,7 +15,7 @@ public class SemanticModel {
private String schemaKey; private String schemaKey;
private List<Metric> metrics = new ArrayList<>(); private List<Metric> metrics = new ArrayList<>();
private Map<String, DataSource> datasourceMap = new HashMap<>(); private Map<String, DataModel> datasourceMap = new HashMap<>();
private Map<String, List<Dimension>> dimensionMap = new HashMap<>(); private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>(); private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
@@ -26,8 +26,8 @@ public class SemanticModel {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
public Map<Long, DataSource> getModelMap() { public Map<Long, DataModel> getModelMap() {
return datasourceMap.values().stream() return datasourceMap.values().stream()
.collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource)); .collect(Collectors.toMap(DataModel::getId, dataSource -> dataSource));
} }
} }

15
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSchema.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/S2SemanticSchema.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema; package com.tencent.supersonic.headless.core.translator.calcite.schema;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
@@ -15,9 +15,10 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
public class SemanticSchema extends AbstractSchema { public class S2SemanticSchema extends AbstractSchema {
private final String schemaKey; private final String schemaKey;
private final Map<String, Table> tableMap; private final Map<String, Table> tableMap;
private SemanticModel semanticModel = new SemanticModel(); private SemanticModel semanticModel = new SemanticModel();
@@ -26,7 +27,7 @@ public class SemanticSchema extends AbstractSchema {
private RuntimeOptions runtimeOptions; private RuntimeOptions runtimeOptions;
private SemanticSchema(String schemaKey, Map<String, Table> tableMap) { private S2SemanticSchema(String schemaKey, Map<String, Table> tableMap) {
this.schemaKey = schemaKey; this.schemaKey = schemaKey;
this.tableMap = tableMap; this.tableMap = tableMap;
} }
@@ -57,11 +58,11 @@ public class SemanticSchema extends AbstractSchema {
return this; return this;
} }
public Map<String, DataSource> getDatasource() { public Map<String, DataModel> getDatasource() {
return semanticModel.getDatasourceMap(); return semanticModel.getDatasourceMap();
} }
public void setDatasource(Map<String, DataSource> datasource) { public void setDatasource(Map<String, DataModel> datasource) {
semanticModel.setDatasourceMap(datasource); semanticModel.setDatasourceMap(datasource);
} }
@@ -129,8 +130,8 @@ public class SemanticSchema extends AbstractSchema {
return this; return this;
} }
public SemanticSchema build() { public S2SemanticSchema build() {
return new SemanticSchema(schemaKey, tableMap); return new S2SemanticSchema(schemaKey, tableMap);
} }
} }
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java
View File

@@ -27,7 +27,7 @@ public class SchemaBuilder {
public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1"; public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1";
public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2"; public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2";
public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception { public static SqlValidatorScope getScope(S2SemanticSchema schema) throws Exception {
Map<String, RelDataType> nameToTypeMap = new HashMap<>(); Map<String, RelDataType> nameToTypeMap = new HashMap<>();
CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false);
rootSchema.add(schema.getSchemaKey(), schema); rootSchema.add(schema.getSchemaKey(), schema);

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Optimization.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql; package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
public interface Optimization { public interface Optimization {
public void visit(SemanticSchema semanticSchema); public void visit(S2SemanticSchema semanticSchema);
} }

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode;
@@ -27,29 +27,29 @@ public abstract class Renderer {
protected TableView tableView = new TableView(); protected TableView tableView = new TableView();
public static Optional<Dimension> getDimensionByName(String name, DataSource datasource) { public static Optional<Dimension> getDimensionByName(String name, DataModel datasource) {
return datasource.getDimensions().stream().filter(d -> d.getName().equalsIgnoreCase(name)) return datasource.getDimensions().stream().filter(d -> d.getName().equalsIgnoreCase(name))
.findFirst(); .findFirst();
} }
public static Optional<Measure> getMeasureByName(String name, DataSource datasource) { public static Optional<Measure> getMeasureByName(String name, DataModel datasource) {
return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)) return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name))
.findFirst(); .findFirst();
} }
public static Optional<Metric> getMetricByName(String name, SemanticSchema schema) { public static Optional<Metric> getMetricByName(String name, S2SemanticSchema schema) {
Optional<Metric> metric = schema.getMetrics().stream() Optional<Metric> metric = schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name)).findFirst(); .filter(m -> m.getName().equalsIgnoreCase(name)).findFirst();
return metric; return metric;
} }
public static Optional<Identify> getIdentifyByName(String name, DataSource datasource) { public static Optional<Identify> getIdentifyByName(String name, DataModel datasource) {
return datasource.getIdentifiers().stream().filter(i -> i.getName().equalsIgnoreCase(name)) return datasource.getIdentifiers().stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst(); .findFirst();
} }
public static MetricNode buildMetricNode(String metric, DataSource datasource, public static MetricNode buildMetricNode(String metric, DataModel datasource,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg, String alias) SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg, String alias)
throws Exception { throws Exception {
Optional<Metric> metricOpt = getMetricByName(metric, schema); Optional<Metric> metricOpt = getMetricByName(metric, schema);
MetricNode metricNode = new MetricNode(); MetricNode metricNode = new MetricNode();
@@ -113,6 +113,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build()); return SemanticNode.buildAs(alias, tableView.build());
} }
public abstract void render(MetricQueryParam metricCommand, List<DataSource> dataSources, public abstract void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception; SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception;
} }

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/TableView.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql; package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import lombok.Data; import lombok.Data;
import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlKind;
@@ -27,7 +27,7 @@ public class TableView {
private String alias; private String alias;
private List<String> primary; private List<String> primary;
private DataSource dataSource; private DataModel dataModel;
public SqlNode build() { public SqlNode build() {
measure.addAll(dimension); measure.addAll(dimension);

122
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java
View File

@@ -6,13 +6,13 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend.LateralViewExplodeNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.extend.LateralViewExplodeNode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlBasicCall;
@@ -40,7 +40,7 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
public class DataSourceNode extends SemanticNode { public class DataSourceNode extends SemanticNode {
public static SqlNode build(DataSource datasource, SqlValidatorScope scope) throws Exception { public static SqlNode build(DataModel datasource, SqlValidatorScope scope) throws Exception {
String sqlTable = ""; String sqlTable = "";
if (datasource.getSqlQuery() != null && !datasource.getSqlQuery().isEmpty()) { if (datasource.getSqlQuery() != null && !datasource.getSqlQuery().isEmpty()) {
sqlTable = datasource.getSqlQuery(); sqlTable = datasource.getSqlQuery();
@@ -61,7 +61,7 @@ public class DataSourceNode extends SemanticNode {
return buildAs(datasource.getName(), source); return buildAs(datasource.getName(), source);
} }
private static void addSchema(SqlValidatorScope scope, DataSource datasource, String table) private static void addSchema(SqlValidatorScope scope, DataModel datasource, String table)
throws Exception { throws Exception {
Map<String, Set<String>> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table); Map<String, Set<String>> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table);
for (Map.Entry<String, Set<String>> entry : sqlTable.entrySet()) { for (Map.Entry<String, Set<String>> entry : sqlTable.entrySet()) {
@@ -75,7 +75,7 @@ public class DataSourceNode extends SemanticNode {
} }
} }
private static void addSchemaTable(SqlValidatorScope scope, DataSource datasource, String db, private static void addSchemaTable(SqlValidatorScope scope, DataModel datasource, String db,
String tb, Set<String> fields) throws Exception { String tb, Set<String> fields) throws Exception {
Set<String> dateInfo = new HashSet<>(); Set<String> dateInfo = new HashSet<>();
Set<String> dimensions = new HashSet<>(); Set<String> dimensions = new HashSet<>();
@@ -112,7 +112,7 @@ public class DataSourceNode extends SemanticNode {
dateInfo, dimensions, metrics); dateInfo, dimensions, metrics);
} }
public static SqlNode buildExtend(DataSource datasource, Map<String, String> exprList, public static SqlNode buildExtend(DataModel datasource, Map<String, String> exprList,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
if (CollectionUtils.isEmpty(exprList)) { if (CollectionUtils.isEmpty(exprList)) {
return build(datasource, scope); return build(datasource, scope);
@@ -146,11 +146,11 @@ public class DataSourceNode extends SemanticNode {
return sqlNode; return sqlNode;
} }
public static String getNames(List<DataSource> dataSourceList) { public static String getNames(List<DataModel> dataModelList) {
return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); return dataModelList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
} }
public static void getQueryDimensionMeasure(SemanticSchema schema, public static void getQueryDimensionMeasure(S2SemanticSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures) { MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream() queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) .map(d -> d.contains(Constants.DIMENSION_IDENTIFY)
@@ -166,7 +166,7 @@ public class DataSourceNode extends SemanticNode {
.forEach(m -> measures.add(m)); .forEach(m -> measures.add(m));
} }
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, public static void mergeQueryFilterDimensionMeasure(S2SemanticSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures, MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType engineType =
@@ -193,18 +193,18 @@ public class DataSourceNode extends SemanticNode {
} }
} }
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, public static List<DataModel> getMatchDataSources(SqlValidatorScope scope,
SemanticSchema schema, MetricQueryParam metricCommand) throws Exception { S2SemanticSchema schema, MetricQueryParam metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>(); List<DataModel> dataModels = new ArrayList<>();
// check by metric // check by metric
List<String> measures = new ArrayList<>(); List<String> measures = new ArrayList<>();
Set<String> queryDimension = new HashSet<>(); Set<String> queryDimension = new HashSet<>();
getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures); getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures);
DataSource baseDataSource = null; DataModel baseDataModel = null;
// one , match measure count // one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>(); Map<String, Integer> dataSourceMeasures = new HashMap<>();
for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) { for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) {
Set<String> sourceMeasure = entry.getValue().getMeasures().stream() Set<String> sourceMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet()); .map(mm -> mm.getName()).collect(Collectors.toSet());
sourceMeasure.retainAll(measures); sourceMeasure.retainAll(measures);
@@ -214,19 +214,19 @@ public class DataSourceNode extends SemanticNode {
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream() Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst(); .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) { if (base.isPresent()) {
baseDataSource = schema.getDatasource().get(base.get().getKey()); baseDataModel = schema.getDatasource().get(base.get().getKey());
dataSources.add(baseDataSource); dataModels.add(baseDataModel);
} }
// second , check match all dimension and metric // second , check match all dimension and metric
if (baseDataSource != null) { if (baseDataModel != null) {
Set<String> filterMeasure = new HashSet<>(); Set<String> filterMeasure = new HashSet<>();
Set<String> sourceMeasure = baseDataSource.getMeasures().stream() Set<String> sourceMeasure = baseDataModel.getMeasures().stream().map(mm -> mm.getName())
.map(mm -> mm.getName()).collect(Collectors.toSet());
Set<String> dimension = baseDataSource.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
baseDataSource.getIdentifiers().stream().forEach(i -> dimension.add(i.getName())); Set<String> dimension = baseDataModel.getDimensions().stream().map(dd -> dd.getName())
if (schema.getDimension().containsKey(baseDataSource.getName())) { .collect(Collectors.toSet());
schema.getDimension().get(baseDataSource.getName()).stream() baseDataModel.getIdentifiers().stream().forEach(i -> dimension.add(i.getName()));
if (schema.getDimension().containsKey(baseDataModel.getName())) {
schema.getDimension().get(baseDataModel.getName()).stream()
.forEach(d -> dimension.add(d.getName())); .forEach(d -> dimension.add(d.getName()));
} }
filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(sourceMeasure);
@@ -238,34 +238,34 @@ public class DataSourceNode extends SemanticNode {
boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension,
metricCommand, scope, engineType); metricCommand, scope, engineType);
if (isAllMatch) { if (isAllMatch) {
log.debug("baseDataSource match all "); log.debug("baseDataModel match all ");
return dataSources; return dataModels;
} }
// find all dataSource has the same identifiers // find all dataSource has the same identifiers
List<DataSource> linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension, List<DataModel> linkDataModels = getLinkDataSourcesByJoinRelation(queryDimension,
measures, baseDataSource, schema); measures, baseDataModel, schema);
if (CollectionUtils.isEmpty(linkDataSources)) { if (CollectionUtils.isEmpty(linkDataModels)) {
log.debug("baseDataSource get by identifiers "); log.debug("baseDataModel get by identifiers ");
Set<String> baseIdentifiers = baseDataSource.getIdentifiers().stream() Set<String> baseIdentifiers = baseDataModel.getIdentifiers().stream()
.map(i -> i.getName()).collect(Collectors.toSet()); .map(i -> i.getName()).collect(Collectors.toSet());
if (baseIdentifiers.isEmpty()) { if (baseIdentifiers.isEmpty()) {
throw new Exception( throw new Exception(
"datasource error : " + baseDataSource.getName() + " miss identifier"); "datasource error : " + baseDataModel.getName() + " miss identifier");
} }
linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures, linkDataModels = getLinkDataSources(baseIdentifiers, queryDimension, measures,
baseDataSource, schema); baseDataModel, schema);
if (linkDataSources.isEmpty()) { if (linkDataModels.isEmpty()) {
throw new Exception(String.format( throw new Exception(String.format(
"not find the match datasource : dimension[%s],measure[%s]", "not find the match datasource : dimension[%s],measure[%s]",
queryDimension, measures)); queryDimension, measures));
} }
} }
log.debug("linkDataSources {}", linkDataSources); log.debug("linkDataModels {}", linkDataModels);
return linkDataSources; return linkDataModels;
// dataSources.addAll(linkDataSources); // dataModels.addAll(linkDataModels);
} }
return dataSources; return dataModels;
} }
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension, private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension,
@@ -301,17 +301,17 @@ public class DataSourceNode extends SemanticNode {
return isAllMatch; return isAllMatch;
} }
private static List<DataSource> getLinkDataSourcesByJoinRelation(Set<String> queryDimension, private static List<DataModel> getLinkDataSourcesByJoinRelation(Set<String> queryDimension,
List<String> measures, DataSource baseDataSource, SemanticSchema schema) { List<String> measures, DataModel baseDataModel, S2SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>(); List<DataModel> linkDataModels = new ArrayList<>();
Set<String> before = new HashSet<>(); Set<String> before = new HashSet<>();
before.add(baseDataSource.getName()); before.add(baseDataModel.getName());
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
Set<Long> visitJoinRelations = new HashSet<>(); Set<Long> visitJoinRelations = new HashSet<>();
List<JoinRelation> sortedJoinRelation = new ArrayList<>(); List<JoinRelation> sortedJoinRelation = new ArrayList<>();
sortJoinRelation(schema.getJoinRelations(), baseDataSource.getName(), sortJoinRelation(schema.getJoinRelations(), baseDataModel.getName(), visitJoinRelations,
visitJoinRelations, sortedJoinRelation); sortedJoinRelation);
schema.getJoinRelations().stream().filter(j -> !visitJoinRelations.contains(j.getId())) schema.getJoinRelations().stream().filter(j -> !visitJoinRelations.contains(j.getId()))
.forEach(j -> sortedJoinRelation.add(j)); .forEach(j -> sortedJoinRelation.add(j));
for (JoinRelation joinRelation : sortedJoinRelation) { for (JoinRelation joinRelation : sortedJoinRelation) {
@@ -321,7 +321,7 @@ public class DataSourceNode extends SemanticNode {
} }
boolean isMatch = false; boolean isMatch = false;
boolean isRight = before.contains(joinRelation.getLeft()); boolean isRight = before.contains(joinRelation.getLeft());
DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight()) DataModel other = isRight ? schema.getDatasource().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft()); : schema.getDatasource().get(joinRelation.getLeft());
if (!queryDimension.isEmpty()) { if (!queryDimension.isEmpty()) {
Set<String> linkDimension = other.getDimensions().stream() Set<String> linkDimension = other.getDimensions().stream()
@@ -354,8 +354,8 @@ public class DataSourceNode extends SemanticNode {
} }
if (!CollectionUtils.isEmpty(linkDataSourceName)) { if (!CollectionUtils.isEmpty(linkDataSourceName)) {
Map<String, Long> orders = new HashMap<>(); Map<String, Long> orders = new HashMap<>();
linkDataSourceName.add(baseDataSource.getName()); linkDataSourceName.add(baseDataModel.getName());
orders.put(baseDataSource.getName(), 0L); orders.put(baseDataModel.getName(), 0L);
for (JoinRelation joinRelation : schema.getJoinRelations()) { for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (linkDataSourceName.contains(joinRelation.getLeft()) if (linkDataSourceName.contains(joinRelation.getLeft())
&& linkDataSourceName.contains(joinRelation.getRight())) { && linkDataSourceName.contains(joinRelation.getRight())) {
@@ -364,10 +364,10 @@ public class DataSourceNode extends SemanticNode {
} }
} }
orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> { orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> {
linkDataSources.add(schema.getDatasource().get(d.getKey())); linkDataModels.add(schema.getDatasource().get(d.getKey()));
}); });
} }
return linkDataSources; return linkDataModels;
} }
private static void sortJoinRelation(List<JoinRelation> joinRelations, String next, private static void sortJoinRelation(List<JoinRelation> joinRelations, String next,
@@ -385,13 +385,13 @@ public class DataSourceNode extends SemanticNode {
} }
} }
private static List<DataSource> getLinkDataSources(Set<String> baseIdentifiers, private static List<DataModel> getLinkDataSources(Set<String> baseIdentifiers,
Set<String> queryDimension, List<String> measures, DataSource baseDataSource, Set<String> queryDimension, List<String> measures, DataModel baseDataModel,
SemanticSchema schema) { S2SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>(); List<DataModel> linkDataModels = new ArrayList<>();
for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) { for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) {
if (entry.getKey().equalsIgnoreCase(baseDataSource.getName())) { if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) {
continue; continue;
} }
Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName()) Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName())
@@ -432,12 +432,12 @@ public class DataSourceNode extends SemanticNode {
} }
} }
for (String linkName : linkDataSourceName) { for (String linkName : linkDataSourceName) {
linkDataSources.add(schema.getDatasource().get(linkName)); linkDataModels.add(schema.getDatasource().get(linkName));
} }
if (!CollectionUtils.isEmpty(linkDataSources)) { if (!CollectionUtils.isEmpty(linkDataModels)) {
List<DataSource> all = new ArrayList<>(); List<DataModel> all = new ArrayList<>();
all.add(baseDataSource); all.add(baseDataModel);
all.addAll(linkDataSources); all.addAll(linkDataModels);
return all; return all;
} }
return Lists.newArrayList(); return Lists.newArrayList();

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import lombok.Data; import lombok.Data;
import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -30,7 +30,7 @@ public class MetricNode extends SemanticNode {
return buildAs(metric.getName(), sqlNode); return buildAs(metric.getName(), sqlNode);
} }
public static Boolean isMetricField(String name, SemanticSchema schema) { public static Boolean isMetricField(String name, S2SemanticSchema schema) {
Optional<Metric> metric = schema.getMetrics().stream() Optional<Metric> metric = schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name)).findFirst(); .filter(m -> m.getName().equalsIgnoreCase(name)).findFirst();
return metric.isPresent() && metric.get().getMetricTypeParams().isFieldMetric(); return metric.isPresent() && metric.get().getMetricTypeParams().isFieldMetric();

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.common.calcite.SemanticSqlDialect;
import com.tencent.supersonic.common.calcite.SqlDialectFactory; import com.tencent.supersonic.common.calcite.SqlDialectFactory;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule; import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptPlanner;
@@ -397,8 +397,8 @@ public abstract class SemanticNode {
return parseInfo; return parseInfo;
} }
public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode, public static SqlNode optimize(SqlValidatorScope scope, S2SemanticSchema schema,
EngineType engineType) { SqlNode sqlNode, EngineType engineType) {
try { try {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType); SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType);

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/optimizer/FilterToGroupScanRule.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer; package com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelRule; import org.apache.calcite.plan.RelRule;
import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Aggregate;
@@ -40,9 +40,10 @@ public class FilterToGroupScanRule extends RelRule<Config> implements Transforma
}); });
}).as(FilterTableScanRule.Config.class); }).as(FilterTableScanRule.Config.class);
private SemanticSchema semanticSchema; private S2SemanticSchema semanticSchema;
public FilterToGroupScanRule(FilterTableScanRule.Config config, SemanticSchema semanticSchema) { public FilterToGroupScanRule(FilterTableScanRule.Config config,
S2SemanticSchema semanticSchema) {
super(config); super(config);
this.semanticSchema = semanticSchema; this.semanticSchema = semanticSchema;
} }

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java
View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode;
@@ -27,8 +27,8 @@ import java.util.stream.Collectors;
public class FilterRender extends Renderer { public class FilterRender extends Renderer {
@Override @Override
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView; TableView tableView = super.tableView;
SqlNode filterNode = null; SqlNode filterNode = null;
List<String> queryMetrics = new ArrayList<>(metricCommand.getMetrics()); List<String> queryMetrics = new ArrayList<>(metricCommand.getMetrics());
@@ -43,9 +43,9 @@ public class FilterRender extends Renderer {
List<String> fieldWhere = whereFields.stream().collect(Collectors.toList()); List<String> fieldWhere = whereFields.stream().collect(Collectors.toList());
Set<String> dimensions = new HashSet<>(); Set<String> dimensions = new HashSet<>();
Set<String> metrics = new HashSet<>(); Set<String> metrics = new HashSet<>();
for (DataSource dataSource : dataSources) { for (DataModel dataModel : dataModels) {
SourceRender.whereDimMetric(fieldWhere, metricCommand.getMetrics(), SourceRender.whereDimMetric(fieldWhere, metricCommand.getMetrics(),
metricCommand.getDimensions(), dataSource, schema, dimensions, metrics); metricCommand.getDimensions(), dataModel, schema, dimensions, metrics);
} }
queryMetrics.addAll(metrics); queryMetrics.addAll(metrics);
queryDimensions.addAll(dimensions); queryDimensions.addAll(dimensions);

125
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -3,13 +3,13 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode;
@@ -48,8 +48,8 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer { public class JoinRender extends Renderer {
@Override @Override
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere(); String queryWhere = metricCommand.getWhere();
EngineType engineType = EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -71,14 +71,14 @@ public class JoinRender extends Renderer {
Set<String> filterDimension = new HashSet<>(); Set<String> filterDimension = new HashSet<>();
Map<String, String> beforeSources = new HashMap<>(); Map<String, String> beforeSources = new HashMap<>();
for (int i = 0; i < dataSources.size(); i++) { for (int i = 0; i < dataModels.size(); i++) {
final DataSource dataSource = dataSources.get(i); final DataModel dataModel = dataModels.get(i);
final Set<String> filterDimensions = new HashSet<>(); final Set<String> filterDimensions = new HashSet<>();
final Set<String> filterMetrics = new HashSet<>(); final Set<String> filterMetrics = new HashSet<>();
final List<String> queryDimension = new ArrayList<>(); final List<String> queryDimension = new ArrayList<>();
final List<String> queryMetrics = new ArrayList<>(); final List<String> queryMetrics = new ArrayList<>();
SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataSource, SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataModel, schema,
schema, filterDimensions, filterMetrics); filterDimensions, filterMetrics);
List<String> reqMetric = new ArrayList<>(metricCommand.getMetrics()); List<String> reqMetric = new ArrayList<>(metricCommand.getMetrics());
reqMetric.addAll(filterMetrics); reqMetric.addAll(filterMetrics);
reqMetric = uniqList(reqMetric); reqMetric = uniqList(reqMetric);
@@ -87,40 +87,40 @@ public class JoinRender extends Renderer {
reqDimension.addAll(filterDimensions); reqDimension.addAll(filterDimensions);
reqDimension = uniqList(reqDimension); reqDimension = uniqList(reqDimension);
Set<String> sourceMeasure = dataSource.getMeasures().stream().map(mm -> mm.getName()) Set<String> sourceMeasure = dataModel.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataSource, sourceMeasure, doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataModel, sourceMeasure,
scope, schema, nonAgg); scope, schema, nonAgg);
Set<String> dimension = dataSource.getDimensions().stream().map(dd -> dd.getName()) Set<String> dimension = dataModel.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataSource, doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataModel,
dimension, scope, schema); dimension, scope, schema);
List<String> primary = new ArrayList<>(); List<String> primary = new ArrayList<>();
for (Identify identify : dataSource.getIdentifiers()) { for (Identify identify : dataModel.getIdentifiers()) {
primary.add(identify.getName()); primary.add(identify.getName());
if (!fieldWhere.contains(identify.getName())) { if (!fieldWhere.contains(identify.getName())) {
fieldWhere.add(identify.getName()); fieldWhere.add(identify.getName());
} }
} }
List<String> dataSourceWhere = new ArrayList<>(fieldWhere); List<String> dataSourceWhere = new ArrayList<>(fieldWhere);
addZipperField(dataSource, dataSourceWhere); addZipperField(dataModel, dataSourceWhere);
TableView tableView = TableView tableView =
SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension, SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension,
metricCommand.getWhere(), dataSources.get(i), scope, schema, true); metricCommand.getWhere(), dataModels.get(i), scope, schema, true);
log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString()));
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName();
tableView.setAlias(alias); tableView.setAlias(alias);
tableView.setPrimary(primary); tableView.setPrimary(primary);
tableView.setDataSource(dataSource); tableView.setDataModel(dataModel);
if (left == null) { if (left == null) {
leftTable = tableView; leftTable = tableView;
left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)); left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope));
beforeSources.put(dataSource.getName(), leftTable.getAlias()); beforeSources.put(dataModel.getName(), leftTable.getAlias());
continue; continue;
} }
left = buildJoin(left, leftTable, tableView, beforeSources, dataSource, schema, scope); left = buildJoin(left, leftTable, tableView, beforeSources, dataModel, schema, scope);
leftTable = tableView; leftTable = tableView;
beforeSources.put(dataSource.getName(), tableView.getAlias()); beforeSources.put(dataModel.getName(), tableView.getAlias());
} }
for (Map.Entry<String, SqlNode> entry : innerSelect.entrySet()) { for (Map.Entry<String, SqlNode> entry : innerSelect.entrySet()) {
@@ -144,16 +144,15 @@ public class JoinRender extends Renderer {
} }
private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView, private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView,
List<String> queryMetrics, List<String> reqMetrics, DataSource dataSource, List<String> queryMetrics, List<String> reqMetrics, DataModel dataModel,
Set<String> sourceMeasure, SqlValidatorScope scope, SemanticSchema schema, Set<String> sourceMeasure, SqlValidatorScope scope, S2SemanticSchema schema,
boolean nonAgg) throws Exception { boolean nonAgg) throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName();
EngineType engineType = EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
for (String m : reqMetrics) { for (String m : reqMetrics) {
if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) {
MetricNode metricNode = MetricNode metricNode = buildMetricNode(m, dataModel, scope, schema, nonAgg, alias);
buildMetricNode(m, dataSource, scope, schema, nonAgg, alias);
if (!metricNode.getNonAggNode().isEmpty()) { if (!metricNode.getNonAggNode().isEmpty()) {
for (String measure : metricNode.getNonAggNode().keySet()) { for (String measure : metricNode.getNonAggNode().keySet()) {
@@ -181,14 +180,14 @@ public class JoinRender extends Renderer {
} }
private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension, private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension,
List<String> queryDimension, List<String> reqDimensions, DataSource dataSource, List<String> queryDimension, List<String> reqDimensions, DataModel dataModel,
Set<String> dimension, SqlValidatorScope scope, SemanticSchema schema) Set<String> dimension, SqlValidatorScope scope, S2SemanticSchema schema)
throws Exception { throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName();
EngineType engineType = EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
for (String d : reqDimensions) { for (String d : reqDimensions) {
if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { if (getMatchDimension(schema, dimension, dataModel, d, queryDimension)) {
if (d.contains(Constants.DIMENSION_IDENTIFY)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) {
String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY); String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY);
innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode
@@ -209,7 +208,7 @@ public class JoinRender extends Renderer {
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }
private boolean getMatchMetric(SemanticSchema schema, Set<String> sourceMeasure, String m, private boolean getMatchMetric(S2SemanticSchema schema, Set<String> sourceMeasure, String m,
List<String> queryMetrics) { List<String> queryMetrics) {
Optional<Metric> metric = schema.getMetrics().stream() Optional<Metric> metric = schema.getMetrics().stream()
.filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst(); .filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst();
@@ -230,8 +229,8 @@ public class JoinRender extends Renderer {
return isAdd; return isAdd;
} }
private boolean getMatchDimension(SemanticSchema schema, Set<String> sourceDimension, private boolean getMatchDimension(S2SemanticSchema schema, Set<String> sourceDimension,
DataSource dataSource, String d, List<String> queryDimension) { DataModel dataModel, String d, List<String> queryDimension) {
String oriDimension = d; String oriDimension = d;
boolean isAdd = false; boolean isAdd = false;
if (d.contains(Constants.DIMENSION_IDENTIFY)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) {
@@ -240,14 +239,14 @@ public class JoinRender extends Renderer {
if (sourceDimension.contains(oriDimension)) { if (sourceDimension.contains(oriDimension)) {
isAdd = true; isAdd = true;
} }
for (Identify identify : dataSource.getIdentifiers()) { for (Identify identify : dataModel.getIdentifiers()) {
if (identify.getName().equalsIgnoreCase(oriDimension)) { if (identify.getName().equalsIgnoreCase(oriDimension)) {
isAdd = true; isAdd = true;
break; break;
} }
} }
if (schema.getDimension().containsKey(dataSource.getName())) { if (schema.getDimension().containsKey(dataModel.getName())) {
for (Dimension dim : schema.getDimension().get(dataSource.getName())) { for (Dimension dim : schema.getDimension().get(dataModel.getName())) {
if (dim.getName().equalsIgnoreCase(oriDimension)) { if (dim.getName().equalsIgnoreCase(oriDimension)) {
isAdd = true; isAdd = true;
} }
@@ -264,12 +263,12 @@ public class JoinRender extends Renderer {
} }
private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView,
Map<String, String> before, DataSource dataSource, SemanticSchema schema, Map<String, String> before, DataModel dataModel, S2SemanticSchema schema,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
SqlNode condition = SqlNode condition =
getCondition(leftTable, tableView, dataSource, schema, scope, engineType); getCondition(leftTable, tableView, dataModel, schema, scope, engineType);
SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral("");
JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema); JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema);
SqlNode joinRelationCondition = null; SqlNode joinRelationCondition = null;
@@ -278,11 +277,11 @@ public class JoinRender extends Renderer {
joinRelationCondition = getCondition(matchJoinRelation, scope, engineType); joinRelationCondition = getCondition(matchJoinRelation, scope, engineType);
condition = joinRelationCondition; condition = joinRelationCondition;
} }
if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataModel().getTimePartType())
|| Materialization.TimePartType.ZIPPER || Materialization.TimePartType.ZIPPER
.equals(tableView.getDataSource().getTimePartType())) { .equals(tableView.getDataModel().getTimePartType())) {
SqlNode zipperCondition = SqlNode zipperCondition =
getZipperCondition(leftTable, tableView, dataSource, schema, scope); getZipperCondition(leftTable, tableView, dataModel, schema, scope);
if (Objects.nonNull(joinRelationCondition)) { if (Objects.nonNull(joinRelationCondition)) {
condition = new SqlBasicCall(SqlStdOperatorTable.AND, condition = new SqlBasicCall(SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(zipperCondition, joinRelationCondition)), new ArrayList<>(Arrays.asList(zipperCondition, joinRelationCondition)),
@@ -299,11 +298,11 @@ public class JoinRender extends Renderer {
} }
private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView, private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView,
SemanticSchema schema) { S2SemanticSchema schema) {
JoinRelation matchJoinRelation = JoinRelation.builder().build(); JoinRelation matchJoinRelation = JoinRelation.builder().build();
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
for (JoinRelation joinRelation : schema.getJoinRelations()) { for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName()) if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataModel().getName())
&& before.containsKey(joinRelation.getLeft())) { && before.containsKey(joinRelation.getLeft())) {
matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream()
.map(r -> Triple.of( .map(r -> Triple.of(
@@ -338,8 +337,8 @@ public class JoinRender extends Renderer {
return condition; return condition;
} }
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, private SqlNode getCondition(TableView left, TableView right, DataModel dataModel,
SemanticSchema schema, SqlValidatorScope scope, EngineType engineType) S2SemanticSchema schema, SqlValidatorScope scope, EngineType engineType)
throws Exception { throws Exception {
Set<String> selectLeft = SemanticNode.getSelect(left.getTable()); Set<String> selectLeft = SemanticNode.getSelect(left.getTable());
@@ -347,16 +346,16 @@ public class JoinRender extends Renderer {
selectLeft.retainAll(selectRight); selectLeft.retainAll(selectRight);
SqlNode condition = null; SqlNode condition = null;
for (String on : selectLeft) { for (String on : selectLeft) {
if (!SourceRender.isDimension(on, dataSource, schema)) { if (!SourceRender.isDimension(on, dataModel, schema)) {
continue; continue;
} }
if (IdentifyNode.isForeign(on, left.getDataSource().getIdentifiers())) { if (IdentifyNode.isForeign(on, left.getDataModel().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, right.getDataSource().getIdentifiers())) { if (!IdentifyNode.isPrimary(on, right.getDataModel().getIdentifiers())) {
continue; continue;
} }
} }
if (IdentifyNode.isForeign(on, right.getDataSource().getIdentifiers())) { if (IdentifyNode.isForeign(on, right.getDataModel().getIdentifiers())) {
if (!IdentifyNode.isPrimary(on, left.getDataSource().getIdentifiers())) { if (!IdentifyNode.isPrimary(on, left.getDataModel().getIdentifiers())) {
continue; continue;
} }
} }
@@ -396,9 +395,9 @@ public class JoinRender extends Renderer {
visited.put(id, false); visited.put(id, false);
} }
private void addZipperField(DataSource dataSource, List<String> fields) { private void addZipperField(DataModel dataModel, List<String> fields) {
if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) {
dataSource.getDimensions().stream() dataModel.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.forEach(t -> { .forEach(t -> {
if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END) if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)
@@ -413,18 +412,18 @@ public class JoinRender extends Renderer {
} }
} }
private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, private SqlNode getZipperCondition(TableView left, TableView right, DataModel dataModel,
SemanticSchema schema, SqlValidatorScope scope) throws Exception { S2SemanticSchema schema, SqlValidatorScope scope) throws Exception {
if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) if (Materialization.TimePartType.ZIPPER.equals(left.getDataModel().getTimePartType())
&& Materialization.TimePartType.ZIPPER && Materialization.TimePartType.ZIPPER
.equals(right.getDataSource().getTimePartType())) { .equals(right.getDataModel().getTimePartType())) {
throw new Exception("not support two zipper table"); throw new Exception("not support two zipper table");
} }
SqlNode condition = null; SqlNode condition = null;
Optional<Dimension> leftTime = left.getDataSource().getDimensions().stream() Optional<Dimension> leftTime = left.getDataModel().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst(); .findFirst();
Optional<Dimension> rightTime = right.getDataSource().getDimensions().stream() Optional<Dimension> rightTime = right.getDataModel().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst(); .findFirst();
if (leftTime.isPresent() && rightTime.isPresent()) { if (leftTime.isPresent() && rightTime.isPresent()) {
@@ -434,7 +433,7 @@ public class JoinRender extends Renderer {
String dateTime = ""; String dateTime = "";
Optional<Dimension> startTimeOp = (Materialization.TimePartType.ZIPPER Optional<Dimension> startTimeOp = (Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource() .equals(left.getDataModel().getTimePartType()) ? left : right).getDataModel()
.getDimensions().stream() .getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME .filter(d -> Constants.DIMENSION_TYPE_TIME
.equalsIgnoreCase(d.getType())) .equalsIgnoreCase(d.getType()))
@@ -442,7 +441,7 @@ public class JoinRender extends Renderer {
.startsWith(Constants.MATERIALIZATION_ZIPPER_START)) .startsWith(Constants.MATERIALIZATION_ZIPPER_START))
.findFirst(); .findFirst();
Optional<Dimension> endTimeOp = (Materialization.TimePartType.ZIPPER Optional<Dimension> endTimeOp = (Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource() .equals(left.getDataModel().getTimePartType()) ? left : right).getDataModel()
.getDimensions().stream() .getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME .filter(d -> Constants.DIMENSION_TYPE_TIME
.equalsIgnoreCase(d.getType())) .equalsIgnoreCase(d.getType()))
@@ -451,11 +450,11 @@ public class JoinRender extends Renderer {
.findFirst(); .findFirst();
if (startTimeOp.isPresent() && endTimeOp.isPresent()) { if (startTimeOp.isPresent() && endTimeOp.isPresent()) {
TableView zipper = Materialization.TimePartType.ZIPPER TableView zipper = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right; .equals(left.getDataModel().getTimePartType()) ? left : right;
TableView partMetric = Materialization.TimePartType.ZIPPER TableView partMetric = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? right : left; .equals(left.getDataModel().getTimePartType()) ? right : left;
Optional<Dimension> partTime = Materialization.TimePartType.ZIPPER Optional<Dimension> partTime = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? rightTime : leftTime; .equals(left.getDataModel().getTimePartType()) ? rightTime : leftTime;
startTime = zipper.getAlias() + "." + startTimeOp.get().getName(); startTime = zipper.getAlias() + "." + startTimeOp.get().getName();
endTime = zipper.getAlias() + "." + endTimeOp.get().getName(); endTime = zipper.getAlias() + "." + endTimeOp.get().getName();
dateTime = partMetric.getAlias() + "." + partTime.get().getName(); dateTime = partMetric.getAlias() + "." + partTime.get().getName();

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode;
@@ -23,8 +23,8 @@ import java.util.List;
public class OutputRender extends Renderer { public class OutputRender extends Renderer {
@Override @Override
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources, public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView; TableView selectDataSet = super.tableView;
EngineType engineType = EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); EngineType.fromString(schema.getSemanticModel().getDatabase().getType());

51
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -3,13 +3,13 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.translator.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataSourceNode;
@@ -21,7 +21,6 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.node.Semantic
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.util.Litmus;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
@@ -44,7 +43,7 @@ public class SourceRender extends Renderer {
public static TableView renderOne(String alias, List<String> fieldWheres, public static TableView renderOne(String alias, List<String> fieldWheres,
List<String> reqMetrics, List<String> reqDimensions, String queryWhere, List<String> reqMetrics, List<String> reqDimensions, String queryWhere,
DataSource datasource, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) DataModel datasource, SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg)
throws Exception { throws Exception {
TableView dataSet = new TableView(); TableView dataSet = new TableView();
@@ -107,8 +106,8 @@ public class SourceRender extends Renderer {
private static void buildDimension(String alias, String dimension, DataSource datasource, private static void buildDimension(String alias, String dimension, DataModel datasource,
SemanticSchema schema, boolean nonAgg, Map<String, String> extendFields, S2SemanticSchema schema, boolean nonAgg, Map<String, String> extendFields,
TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception { TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName()); List<Dimension> dimensionList = schema.getDimension().get(datasource.getName());
EngineType engineType = EngineType engineType =
@@ -186,8 +185,8 @@ public class SourceRender extends Renderer {
} }
private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics, private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, Map<String, String> extendFields, DataSource datasource, List<String> queryDimensions, Map<String, String> extendFields, DataModel datasource,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception {
Iterator<String> iterator = fields.iterator(); Iterator<String> iterator = fields.iterator();
List<SqlNode> whereNode = new ArrayList<>(); List<SqlNode> whereNode = new ArrayList<>();
EngineType engineType = EngineType engineType =
@@ -229,8 +228,8 @@ public class SourceRender extends Renderer {
private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet, private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet,
List<String> queryMetrics, List<String> queryDimensions, List<String> queryMetrics, List<String> queryDimensions,
Map<String, String> extendFields, DataSource datasource, SqlValidatorScope scope, Map<String, String> extendFields, DataModel datasource, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception { S2SemanticSchema schema, boolean nonAgg) throws Exception {
List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions,
extendFields, datasource, scope, schema, nonAgg); extendFields, datasource, scope, schema, nonAgg);
dataSet.getMeasure().addAll(whereNode); dataSet.getMeasure().addAll(whereNode);
@@ -238,7 +237,7 @@ public class SourceRender extends Renderer {
} }
public static void whereDimMetric(List<String> fields, List<String> queryMetrics, public static void whereDimMetric(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, DataSource datasource, SemanticSchema schema, List<String> queryDimensions, DataModel datasource, S2SemanticSchema schema,
Set<String> dimensions, Set<String> metrics) { Set<String> dimensions, Set<String> metrics) {
for (String field : fields) { for (String field : fields) {
if (queryDimensions.contains(field) || queryMetrics.contains(field)) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) {
@@ -252,8 +251,8 @@ public class SourceRender extends Renderer {
} }
} }
private static void addField(String field, String oriField, DataSource datasource, private static void addField(String field, String oriField, DataModel datasource,
SemanticSchema schema, Set<String> dimensions, Set<String> metrics) { S2SemanticSchema schema, Set<String> dimensions, Set<String> metrics) {
Optional<Dimension> dimension = datasource.getDimensions().stream() Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); .filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dimension.isPresent()) { if (dimension.isPresent()) {
@@ -293,7 +292,7 @@ public class SourceRender extends Renderer {
} }
} }
public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) { public static boolean isDimension(String name, DataModel datasource, S2SemanticSchema schema) {
Optional<Dimension> dimension = datasource.getDimensions().stream() Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dimension.isPresent()) { if (dimension.isPresent()) {
@@ -314,13 +313,13 @@ public class SourceRender extends Renderer {
return false; return false;
} }
private static void addTimeDimension(DataSource dataSource, List<String> queryDimension) { private static void addTimeDimension(DataModel dataModel, List<String> queryDimension) {
if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) { if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) {
Optional<Dimension> startTimeOp = dataSource.getDimensions().stream() Optional<Dimension> startTimeOp = dataModel.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)) .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START))
.findFirst(); .findFirst();
Optional<Dimension> endTimeOp = dataSource.getDimensions().stream() Optional<Dimension> endTimeOp = dataModel.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)) .filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END))
.findFirst(); .findFirst();
@@ -331,7 +330,7 @@ public class SourceRender extends Renderer {
queryDimension.add(endTimeOp.get().getName()); queryDimension.add(endTimeOp.get().getName());
} }
} else { } else {
Optional<Dimension> timeOp = dataSource.getDimensions().stream() Optional<Dimension> timeOp = dataModel.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst(); .findFirst();
if (timeOp.isPresent() && !queryDimension.contains(timeOp.get().getName())) { if (timeOp.isPresent() && !queryDimension.contains(timeOp.get().getName())) {
@@ -340,8 +339,8 @@ public class SourceRender extends Renderer {
} }
} }
public void render(MetricQueryParam metricQueryParam, List<DataSource> dataSources, public void render(MetricQueryParam metricQueryParam, List<DataModel> dataModels,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryParam.getWhere(); String queryWhere = metricQueryParam.getWhere();
Set<String> whereFields = new HashSet<>(); Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>(); List<String> fieldWhere = new ArrayList<>();
@@ -352,15 +351,15 @@ public class SourceRender extends Renderer {
FilterNode.getFilterField(sqlNode, whereFields); FilterNode.getFilterField(sqlNode, whereFields);
fieldWhere = whereFields.stream().collect(Collectors.toList()); fieldWhere = whereFields.stream().collect(Collectors.toList());
} }
if (dataSources.size() == 1) { if (dataModels.size() == 1) {
DataSource dataSource = dataSources.get(0); DataModel dataModel = dataModels.get(0);
super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(), super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataSource, metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataModel, scope,
scope, schema, nonAgg); schema, nonAgg);
return; return;
} }
JoinRender joinRender = new JoinRender(); JoinRender joinRender = new JoinRender();
joinRender.render(metricQueryParam, dataSources, scope, schema, nonAgg); joinRender.render(metricQueryParam, dataModels, scope, schema, nonAgg);
super.tableView = joinRender.getTableView(); super.tableView = joinRender.getTableView();
} }
} }

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
@@ -60,7 +60,7 @@ public class ParserDefaultConverter implements QueryConverter {
// support detail query // support detail query
if (queryParam.getQueryType().isNativeAggQuery() if (queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataSource> modelMap = queryStatement.getSemanticModel().getModelMap(); Map<Long, DataModel> modelMap = queryStatement.getSemanticModel().getModelMap();
for (Long modelId : modelMap.keySet()) { for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName(); String modelBizName = modelMap.get(modelId).getName();
String internalMetricName = String internalMetricName =

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -39,9 +39,9 @@ public class SqlVariableParseConverter implements QueryConverter {
SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(),
modelResp.getModelDetail().getSqlVariables(), modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams()); queryStatement.getQueryParam().getParams());
DataSource dataSource = queryStatement.getSemanticModel().getDatasourceMap() DataModel dataModel = queryStatement.getSemanticModel().getDatasourceMap()
.get(modelResp.getBizName()); .get(modelResp.getBizName());
dataSource.setSqlQuery(sqlParsed); dataModel.setSqlQuery(sqlParsed);
} }
} }
} }

23
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams;
@@ -19,7 +19,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
@@ -73,9 +73,9 @@ public class SemanticSchemaManager {
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
} }
if (!dataModelYamlTpls.isEmpty()) { if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = Map<String, DataModel> dataSourceMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect( dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect(
Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap); semanticModel.setDatasourceMap(dataSourceMap);
} }
if (!dimensionYamlTpls.isEmpty()) { if (!dimensionYamlTpls.isEmpty()) {
@@ -107,8 +107,7 @@ public class SemanticSchemaManager {
} }
if (Objects.nonNull(semanticModel.getDatasourceMap()) if (Objects.nonNull(semanticModel.getDatasourceMap())
&& !semanticModel.getDatasourceMap().isEmpty()) { && !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap() for (Map.Entry<String, DataModel> entry : semanticModel.getDatasourceMap().entrySet()) {
.entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>(); List<Dimension> modelDimensions = new ArrayList<>();
if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) { if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions); semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions);
@@ -178,8 +177,8 @@ public class SemanticSchemaManager {
return getDimension(t); return getDimension(t);
} }
public static DataSource getDatasource(final DataModelYamlTpl d) { public static DataModel getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId()) DataModel datasource = DataModel.builder().id(d.getId()).sourceId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()) .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures())) .measures(getMeasureParams(d.getMeasures()))
@@ -356,17 +355,17 @@ public class SemanticSchemaManager {
return joinRelations; return joinRelations;
} }
public static void update(SemanticSchema schema, List<Metric> metric) throws Exception { public static void update(S2SemanticSchema schema, List<Metric> metric) throws Exception {
if (schema != null) { if (schema != null) {
updateMetric(metric, schema.getMetrics()); updateMetric(metric, schema.getMetrics());
} }
} }
public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) public static void update(S2SemanticSchema schema, DataModel datasourceYamlTpl)
throws Exception { throws Exception {
if (schema != null) { if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName(); String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataSource>> datasourceYamlTplMap = Optional<Entry<String, DataModel>> datasourceYamlTplMap =
schema.getDatasource().entrySet().stream() schema.getDatasource().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst(); .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) { if (datasourceYamlTplMap.isPresent()) {
@@ -377,7 +376,7 @@ public class SemanticSchemaManager {
} }
} }
public static void update(SemanticSchema schema, String datasourceBizName, public static void update(S2SemanticSchema schema, String datasourceBizName,
List<Dimension> dimensionYamlTpls) throws Exception { List<Dimension> dimensionYamlTpls) throws Exception {
if (schema != null) { if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema

12
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
@@ -27,9 +27,9 @@ import java.util.Map;
@Slf4j @Slf4j
class HeadlessParserServiceTest { class HeadlessParserServiceTest {
private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>(); private static Map<String, S2SemanticSchema> headlessSchemaMap = new HashMap<>();
public static SqlParserResp parser(SemanticSchema semanticSchema, public static SqlParserResp parser(S2SemanticSchema semanticSchema,
MetricQueryParam metricQueryParam, boolean isAgg) { MetricQueryParam metricQueryParam, boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp(); SqlParserResp sqlParser = new SqlParserResp();
try { try {
@@ -40,7 +40,7 @@ class HeadlessParserServiceTest {
AggPlanner aggBuilder = new AggPlanner(semanticSchema); AggPlanner aggBuilder = new AggPlanner(semanticSchema);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricQueryParam(metricQueryParam); queryStatement.setMetricQueryParam(metricQueryParam);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); aggBuilder.plan(queryStatement, AggOption.getAggregation(!isAgg));
EngineType engineType = EngineType EngineType engineType = EngineType
.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); .fromString(semanticSchema.getSemanticModel().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType)); sqlParser.setSql(aggBuilder.getSql(engineType));
@@ -122,7 +122,7 @@ class HeadlessParserServiceTest {
identify.setType("primary"); identify.setType("primary");
identifies.add(identify); identifies.add(identify);
datasource.setIdentifiers(identifies); datasource.setIdentifiers(identifies);
SemanticSchema semanticSchema = SemanticSchema.newBuilder("1").build(); S2SemanticSchema semanticSchema = S2SemanticSchema.newBuilder("1").build();
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.update(semanticSchema,
SemanticSchemaManager.getDatasource(datasource)); SemanticSchemaManager.getDatasource(datasource));
@@ -192,7 +192,7 @@ class HeadlessParserServiceTest {
System.out.println(parser(semanticSchema, metricCommand2, true)); System.out.println(parser(semanticSchema, metricCommand2, true));
} }
private static void addDepartment(SemanticSchema semanticSchema) { private static void addDepartment(S2SemanticSchema semanticSchema) {
DataModelYamlTpl datasource = new DataModelYamlTpl(); DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("user_department"); datasource.setName("user_department");
datasource.setSourceId(1L); datasource.setSourceId(1L);

3
launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java
View File

@@ -50,8 +50,7 @@ public class SchemaAuthTest extends BaseTest {
@Test @Test
public void test_getVisibleModelList_alice() { public void test_getVisibleModelList_alice() {
User user = DataUtils.getUserAlice(); User user = DataUtils.getUserAlice();
List<ModelResp> modelResps = List<ModelResp> modelResps = modelService.getModelListWithAuth(user, null, AuthType.VIEWER);
modelService.getModelListWithAuth(user, null, AuthType.VIEWER);
List<String> expectedModelBizNames = Lists.newArrayList("user_department", "singer"); List<String> expectedModelBizNames = Lists.newArrayList("user_department", "singer");
Assertions.assertEquals(expectedModelBizNames, Assertions.assertEquals(expectedModelBizNames,
modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList())); modelResps.stream().map(ModelResp::getBizName).collect(Collectors.toList()));