topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Refactor HeadlessModel-related code, remove modelSchemaResps. (#647)

Browse Source
This commit is contained in:
lexluo09
2024-01-18 11:41:27 +08:00
committed by GitHub
parent d4eecc1bf8
commit dfb8e3a427
25 changed files with 155 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -30,7 +30,7 @@ public class JdbcExecutor implements QueryExecutor {
return null; return null;
} }
log.info("query SQL: {}", queryStatement.getSql()); log.info("query SQL: {}", queryStatement.getSql());
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp();
log.info("database info:{}", databaseResp); log.info("database info:{}", databaseResp);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java
View File

@@ -128,7 +128,7 @@ public class QueryParser {
tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds()); tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setHeadlessModel(queryStatement.getHeadlessModel()); tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) { if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql()); tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewAlias(metricTable.getAlias()); tableSql.setViewAlias(metricTable.getAlias());

28
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -4,8 +4,8 @@ import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions; import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import java.util.Objects; import java.util.Objects;
@@ -22,14 +22,14 @@ public class CalciteSqlParser implements SqlParser {
@Override @Override
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception { public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
MetricQueryReq metricReq = queryStatement.getMetricReq(); MetricQueryReq metricReq = queryStatement.getMetricReq();
HeadlessModel headlessModel = queryStatement.getHeadlessModel(); SemanticModel semanticModel = queryStatement.getSemanticModel();
if (headlessModel == null) { if (semanticModel == null) {
queryStatement.setErrMsg("semanticSchema not found"); queryStatement.setErrMsg("semanticSchema not found");
return queryStatement; return queryStatement;
} }
queryStatement.setMetricReq(metricReq); queryStatement.setMetricReq(metricReq);
HeadlessSchema headlessSchema = getSemanticSchema(headlessModel, queryStatement); SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement);
AggPlanner aggBuilder = new AggPlanner(headlessSchema); AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, isAgg); aggBuilder.explain(queryStatement, isAgg);
queryStatement.setSql(aggBuilder.getSql()); queryStatement.setSql(aggBuilder.getSql());
queryStatement.setSourceId(aggBuilder.getSourceId()); queryStatement.setSourceId(aggBuilder.getSourceId());
@@ -46,15 +46,15 @@ public class CalciteSqlParser implements SqlParser {
return queryStatement; return queryStatement;
} }
private HeadlessSchema getSemanticSchema(HeadlessModel headlessModel, QueryStatement queryStatement) { private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) {
HeadlessSchema headlessSchema = HeadlessSchema.newBuilder(headlessModel.getRootPath()).build(); SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build();
headlessSchema.setDatasource(headlessModel.getDatasourceMap()); semanticSchema.setDatasource(semanticModel.getDatasourceMap());
headlessSchema.setDimension(headlessModel.getDimensionMap()); semanticSchema.setDimension(semanticModel.getDimensionMap());
headlessSchema.setMetric(headlessModel.getMetrics()); semanticSchema.setMetric(semanticModel.getMetrics());
headlessSchema.setJoinRelations(headlessModel.getJoinRelations()); semanticSchema.setJoinRelations(semanticModel.getJoinRelations());
headlessSchema.setRuntimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) semanticSchema.setRuntimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime())
.enableOptimize(queryStatement.getEnableOptimize()).build()); .enableOptimize(queryStatement.getEnableOptimize()).build());
return headlessSchema; return semanticSchema;
} }
private String getSqlByView(String sql, String viewSql, String viewAlias) { private String getSqlByView(String sql, String viewSql, String viewAlias) {

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
@@ -34,7 +34,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
public class AggPlanner implements Planner { public class AggPlanner implements Planner {
private MetricQueryReq metricReq; private MetricQueryReq metricReq;
private HeadlessSchema schema; private SemanticSchema schema;
private SqlValidatorScope scope; private SqlValidatorScope scope;
private Stack<TableView> dataSets = new Stack<>(); private Stack<TableView> dataSets = new Stack<>();
private SqlNode parserNode; private SqlNode parserNode;
@@ -42,7 +42,7 @@ public class AggPlanner implements Planner {
private boolean isAgg = false; private boolean isAgg = false;
private AggOption aggOption = AggOption.DEFAULT; private AggOption aggOption = AggOption.DEFAULT;
public AggPlanner(HeadlessSchema schema) { public AggPlanner(SemanticSchema schema) {
this.schema = schema; this.schema = schema;
} }

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticItem; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticItem;
import java.util.List;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@@ -15,8 +16,9 @@ public class Dimension implements SemanticItem {
private String type; private String type;
private String expr; private String expr;
private DimensionTimeTypeParams dimensionTimeTypeParams; private DimensionTimeTypeParams dimensionTimeTypeParams;
private DataType dataType = DataType.UNKNOWN; private DataType dataType = DataType.UNKNOWN;
private String bizName;
private List<String> defaultValues;
@Override @Override
public String getName() { public String getName() {

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java
View File

@@ -2,18 +2,16 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import lombok.Data;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.Data;
@Data @Data
public class HeadlessModel { public class SemanticModel {
private String rootPath; private String rootPath;
private List<Metric> metrics = new ArrayList<>(); private List<Metric> metrics = new ArrayList<>();
@@ -23,7 +21,6 @@ public class HeadlessModel {
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
private ItemDateResp dataDate; private ItemDateResp dataDate;
private DatabaseResp databaseResp; private DatabaseResp databaseResp;
private List<ModelSchemaResp> modelSchemaResps;
public List<Dimension> getDimensions() { public List<Dimension> getDimensions() {
return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java
View File

@@ -27,7 +27,7 @@ public class SchemaBuilder {
public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1"; public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1";
public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2"; public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2";
public static SqlValidatorScope getScope(HeadlessSchema schema) throws Exception { public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception {
Map<String, RelDataType> nameToTypeMap = new HashMap<>(); Map<String, RelDataType> nameToTypeMap = new HashMap<>();
CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false);
rootSchema.add(schema.getRootPath(), schema); rootSchema.add(schema.getRootPath(), schema);

36
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSchema.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaVersion; import org.apache.calcite.schema.SchemaVersion;
import org.apache.calcite.schema.Table; import org.apache.calcite.schema.Table;
@@ -17,19 +17,19 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class HeadlessSchema extends AbstractSchema { public class SemanticSchema extends AbstractSchema {
private final String rootPath; private final String rootPath;
private final Map<String, Table> tableMap; private final Map<String, Table> tableMap;
private HeadlessModel headlessModel = new HeadlessModel(); private SemanticModel semanticModel = new SemanticModel();
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
private RuntimeOptions runtimeOptions; private RuntimeOptions runtimeOptions;
private HeadlessSchema(String rootPath, Map<String, Table> tableMap) { private SemanticSchema(String rootPath, Map<String, Table> tableMap) {
this.rootPath = rootPath; this.rootPath = rootPath;
this.tableMap = tableMap; this.tableMap = tableMap;
} }
@@ -42,12 +42,12 @@ public class HeadlessSchema extends AbstractSchema {
return rootPath; return rootPath;
} }
public void setSemanticModel(HeadlessModel headlessModel) { public void setSemanticModel(SemanticModel semanticModel) {
this.headlessModel = headlessModel; this.semanticModel = semanticModel;
} }
public HeadlessModel getSemanticModel() { public SemanticModel getSemanticModel() {
return headlessModel; return semanticModel;
} }
@Override @Override
@@ -61,35 +61,35 @@ public class HeadlessSchema extends AbstractSchema {
} }
public Map<String, DataSource> getDatasource() { public Map<String, DataSource> getDatasource() {
return headlessModel.getDatasourceMap(); return semanticModel.getDatasourceMap();
} }
public void setDatasource(Map<String, DataSource> datasource) { public void setDatasource(Map<String, DataSource> datasource) {
headlessModel.setDatasourceMap(datasource); semanticModel.setDatasourceMap(datasource);
} }
public Map<String, List<Dimension>> getDimension() { public Map<String, List<Dimension>> getDimension() {
return headlessModel.getDimensionMap(); return semanticModel.getDimensionMap();
} }
public void setDimension(Map<String, List<Dimension>> dimensions) { public void setDimension(Map<String, List<Dimension>> dimensions) {
headlessModel.setDimensionMap(dimensions); semanticModel.setDimensionMap(dimensions);
} }
public List<Metric> getMetrics() { public List<Metric> getMetrics() {
return headlessModel.getMetrics(); return semanticModel.getMetrics();
} }
public void setMetric(List<Metric> metric) { public void setMetric(List<Metric> metric) {
headlessModel.setMetrics(metric); semanticModel.setMetrics(metric);
} }
public void setMaterializationList(List<Materialization> materializationList) { public void setMaterializationList(List<Materialization> materializationList) {
headlessModel.setMaterializationList(materializationList); semanticModel.setMaterializationList(materializationList);
} }
public List<Materialization> getMaterializationList() { public List<Materialization> getMaterializationList() {
return headlessModel.getMaterializationList(); return semanticModel.getMaterializationList();
} }
public void setJoinRelations(List<JoinRelation> joinRelations) { public void setJoinRelations(List<JoinRelation> joinRelations) {
@@ -131,8 +131,8 @@ public class HeadlessSchema extends AbstractSchema {
return this; return this;
} }
public HeadlessSchema build() { public SemanticSchema build() {
return new HeadlessSchema(rootPath, tableMap); return new SemanticSchema(rootPath, tableMap);
} }
} }

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Optimization.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.parser.calcite.sql; package com.tencent.supersonic.headless.core.parser.calcite.sql;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
public interface Optimization { public interface Optimization {
public void visit(HeadlessSchema headlessSchema); public void visit(SemanticSchema semanticSchema);
} }

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@@ -36,7 +36,7 @@ public abstract class Renderer {
return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)).findFirst(); return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)).findFirst();
} }
public static Optional<Metric> getMetricByName(String name, HeadlessSchema schema) { public static Optional<Metric> getMetricByName(String name, SemanticSchema schema) {
Optional<Metric> metric = schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(name)) Optional<Metric> metric = schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(name))
.findFirst(); .findFirst();
return metric; return metric;
@@ -47,7 +47,7 @@ public abstract class Renderer {
} }
public static MetricNode buildMetricNode(String metric, DataSource datasource, SqlValidatorScope scope, public static MetricNode buildMetricNode(String metric, DataSource datasource, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg, String alias) throws Exception { SemanticSchema schema, boolean nonAgg, String alias) throws Exception {
Optional<Metric> metricOpt = getMetricByName(metric, schema); Optional<Metric> metricOpt = getMetricByName(metric, schema);
MetricNode metricNode = new MetricNode(); MetricNode metricNode = new MetricNode();
if (metricOpt.isPresent()) { if (metricOpt.isPresent()) {
@@ -105,5 +105,5 @@ public abstract class Renderer {
} }
public abstract void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public abstract void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception; SemanticSchema schema, boolean nonAgg) throws Exception;
} }

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import java.util.ArrayList; import java.util.ArrayList;
@@ -149,7 +149,7 @@ public class DataSourceNode extends SemanticNode {
return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
} }
public static void getQueryDimensionMeasure(HeadlessSchema schema, MetricQueryReq metricCommand, public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand,
Set<String> queryDimension, List<String> measures) { Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream() queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d) .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d)
@@ -161,7 +161,7 @@ public class DataSourceNode extends SemanticNode {
} }
public static void mergeQueryFilterDimensionMeasure(HeadlessSchema schema, MetricQueryReq metricCommand, public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand,
Set<String> queryDimension, List<String> measures, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
@@ -184,7 +184,7 @@ public class DataSourceNode extends SemanticNode {
} }
} }
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, HeadlessSchema schema, public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema,
MetricQueryReq metricCommand) throws Exception { MetricQueryReq metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>(); List<DataSource> dataSources = new ArrayList<>();
@@ -283,7 +283,7 @@ public class DataSourceNode extends SemanticNode {
} }
private static List<DataSource> getLinkDataSourcesByJoinRelation(Set<String> queryDimension, List<String> measures, private static List<DataSource> getLinkDataSourcesByJoinRelation(Set<String> queryDimension, List<String> measures,
DataSource baseDataSource, HeadlessSchema schema) { DataSource baseDataSource, SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>(); List<DataSource> linkDataSources = new ArrayList<>();
Set<String> before = new HashSet<>(); Set<String> before = new HashSet<>();
@@ -370,7 +370,7 @@ public class DataSourceNode extends SemanticNode {
Set<String> queryDimension, Set<String> queryDimension,
List<String> measures, List<String> measures,
DataSource baseDataSource, DataSource baseDataSource,
HeadlessSchema schema) { SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>(); List<DataSource> linkDataSources = new ArrayList<>();
for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) { for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) {

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule; import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule;
import java.util.ArrayList; import java.util.ArrayList;
@@ -335,7 +335,7 @@ public abstract class SemanticNode {
return parseInfo; return parseInfo;
} }
public static SqlNode optimize(SqlValidatorScope scope, HeadlessSchema schema, SqlNode sqlNode) { public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode) {
try { try {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/optimizer/FilterToGroupScanRule.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer; package com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
@@ -44,23 +44,23 @@ public class FilterToGroupScanRule extends RelRule<Config>
}).as(FilterTableScanRule.Config.class); }).as(FilterTableScanRule.Config.class);
private HeadlessSchema headlessSchema; private SemanticSchema semanticSchema;
public FilterToGroupScanRule(FilterTableScanRule.Config config, HeadlessSchema headlessSchema) { public FilterToGroupScanRule(FilterTableScanRule.Config config, SemanticSchema semanticSchema) {
super(config); super(config);
this.headlessSchema = headlessSchema; this.semanticSchema = semanticSchema;
} }
public void onMatch(RelOptRuleCall call) { public void onMatch(RelOptRuleCall call) {
if (call.rels.length != 4) { if (call.rels.length != 4) {
return; return;
} }
if (Objects.isNull(headlessSchema.getRuntimeOptions()) || Objects.isNull( if (Objects.isNull(semanticSchema.getRuntimeOptions()) || Objects.isNull(
headlessSchema.getRuntimeOptions().getMinMaxTime()) || headlessSchema.getRuntimeOptions() semanticSchema.getRuntimeOptions().getMinMaxTime()) || semanticSchema.getRuntimeOptions()
.getMinMaxTime().getLeft().isEmpty()) { .getMinMaxTime().getLeft().isEmpty()) {
return; return;
} }
Triple<String, String, String> minMax = headlessSchema.getRuntimeOptions().getMinMaxTime(); Triple<String, String, String> minMax = semanticSchema.getRuntimeOptions().getMinMaxTime();
Filter filter = (Filter) call.rel(0); Filter filter = (Filter) call.rel(0);
Project project0 = (Project) call.rel(1); Project project0 = (Project) call.rel(1);
Project project1 = (Project) call.rel(3); Project project1 = (Project) call.rel(3);

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.FilterNode;
@@ -30,7 +30,7 @@ public class FilterRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView; TableView tableView = super.tableView;
SqlNode filterNode = null; SqlNode filterNode = null;
List<String> queryMetrics = new ArrayList<>(metricCommand.getMetrics()); List<String> queryMetrics = new ArrayList<>(metricCommand.getMetrics());

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.AggFunctionNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.AggFunctionNode;
@@ -49,7 +49,7 @@ public class JoinRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere(); String queryWhere = metricCommand.getWhere();
//dataSources = getOrderSource(dataSources); //dataSources = getOrderSource(dataSources);
Set<String> whereFields = new HashSet<>(); Set<String> whereFields = new HashSet<>();
@@ -141,7 +141,7 @@ public class JoinRender extends Renderer {
private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView, List<String> queryMetrics, private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView, List<String> queryMetrics,
List<String> reqMetrics, DataSource dataSource, Set<String> sourceMeasure, SqlValidatorScope scope, List<String> reqMetrics, DataSource dataSource, Set<String> sourceMeasure, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
for (String m : reqMetrics) { for (String m : reqMetrics) {
if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) {
@@ -175,7 +175,7 @@ public class JoinRender extends Renderer {
private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension, List<String> queryDimension, private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension, List<String> queryDimension,
List<String> reqDimensions, DataSource dataSource, Set<String> dimension, SqlValidatorScope scope, List<String> reqDimensions, DataSource dataSource, Set<String> dimension, SqlValidatorScope scope,
HeadlessSchema schema) throws Exception { SemanticSchema schema) throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
for (String d : reqDimensions) { for (String d : reqDimensions) {
if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) {
@@ -198,7 +198,7 @@ public class JoinRender extends Renderer {
Collectors.toSet()); Collectors.toSet());
} }
private boolean getMatchMetric(HeadlessSchema schema, Set<String> sourceMeasure, String m, private boolean getMatchMetric(SemanticSchema schema, Set<String> sourceMeasure, String m,
List<String> queryMetrics) { List<String> queryMetrics) {
Optional<Metric> metric = schema.getMetrics().stream().filter(mm -> mm.getName().equalsIgnoreCase(m)) Optional<Metric> metric = schema.getMetrics().stream().filter(mm -> mm.getName().equalsIgnoreCase(m))
.findFirst(); .findFirst();
@@ -219,7 +219,7 @@ public class JoinRender extends Renderer {
return isAdd; return isAdd;
} }
private boolean getMatchDimension(HeadlessSchema schema, Set<String> sourceDimension, DataSource dataSource, private boolean getMatchDimension(SemanticSchema schema, Set<String> sourceDimension, DataSource dataSource,
String d, List<String> queryDimension) { String d, List<String> queryDimension) {
String oriDimension = d; String oriDimension = d;
boolean isAdd = false; boolean isAdd = false;
@@ -254,7 +254,7 @@ public class JoinRender extends Renderer {
private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map<String, String> before, private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map<String, String> before,
DataSource dataSource, DataSource dataSource,
HeadlessSchema schema, SqlValidatorScope scope) SemanticSchema schema, SqlValidatorScope scope)
throws Exception { throws Exception {
SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope); SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope);
SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral("");
@@ -289,7 +289,7 @@ public class JoinRender extends Renderer {
); );
} }
private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView, HeadlessSchema schema) { private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView, SemanticSchema schema) {
JoinRelation matchJoinRelation = JoinRelation.builder().build(); JoinRelation matchJoinRelation = JoinRelation.builder().build();
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
for (JoinRelation joinRelation : schema.getJoinRelations()) { for (JoinRelation joinRelation : schema.getJoinRelations()) {
@@ -333,7 +333,7 @@ public class JoinRender extends Renderer {
return condition; return condition;
} }
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, HeadlessSchema schema, private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
Set<String> selectLeft = SemanticNode.getSelect(left.getTable()); Set<String> selectLeft = SemanticNode.getSelect(left.getTable());
@@ -413,7 +413,7 @@ public class JoinRender extends Renderer {
} }
} }
private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, HeadlessSchema schema, private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType())
&& Materialization.TimePartType.ZIPPER.equals(right.getDataSource().getTimePartType())) { && Materialization.TimePartType.ZIPPER.equals(right.getDataSource().getTimePartType())) {

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java
View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode;
@@ -25,7 +25,7 @@ public class OutputRender extends Renderer {
@Override @Override
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView; TableView selectDataSet = super.tableView;
for (String dimension : metricCommand.getDimensions()) { for (String dimension : metricCommand.getDimensions()) {
selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope)); selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope));

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode;
@@ -41,7 +41,7 @@ public class SourceRender extends Renderer {
public static TableView renderOne(String alias, List<String> fieldWheres, public static TableView renderOne(String alias, List<String> fieldWheres,
List<String> reqMetrics, List<String> reqDimensions, List<String> reqMetrics, List<String> reqDimensions,
String queryWhere, DataSource datasource, SqlValidatorScope scope, String queryWhere, DataSource datasource, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
TableView dataSet = new TableView(); TableView dataSet = new TableView();
TableView output = new TableView(); TableView output = new TableView();
@@ -93,7 +93,7 @@ public class SourceRender extends Renderer {
return output; return output;
} }
private static void buildDimension(String alias, String dimension, DataSource datasource, HeadlessSchema schema, private static void buildDimension(String alias, String dimension, DataSource datasource, SemanticSchema schema,
boolean nonAgg, Set<String> extendFields, TableView dataSet, TableView output, SqlValidatorScope scope) boolean nonAgg, Set<String> extendFields, TableView dataSet, TableView output, SqlValidatorScope scope)
throws Exception { throws Exception {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName()); List<Dimension> dimensionList = schema.getDimension().get(datasource.getName());
@@ -161,7 +161,7 @@ public class SourceRender extends Renderer {
private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics, private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, Set<String> extendFields, DataSource datasource, SqlValidatorScope scope, List<String> queryDimensions, Set<String> extendFields, DataSource datasource, SqlValidatorScope scope,
HeadlessSchema schema, SemanticSchema schema,
boolean nonAgg) throws Exception { boolean nonAgg) throws Exception {
Iterator<String> iterator = fields.iterator(); Iterator<String> iterator = fields.iterator();
List<SqlNode> whereNode = new ArrayList<>(); List<SqlNode> whereNode = new ArrayList<>();
@@ -205,7 +205,7 @@ public class SourceRender extends Renderer {
private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet, private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet,
List<String> queryMetrics, List<String> queryMetrics,
List<String> queryDimensions, Set<String> extendFields, DataSource datasource, SqlValidatorScope scope, List<String> queryDimensions, Set<String> extendFields, DataSource datasource, SqlValidatorScope scope,
HeadlessSchema schema, SemanticSchema schema,
boolean nonAgg) throws Exception { boolean nonAgg) throws Exception {
List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource,
scope, schema, scope, schema,
@@ -215,7 +215,7 @@ public class SourceRender extends Renderer {
} }
public static void whereDimMetric(List<String> fields, List<String> queryMetrics, public static void whereDimMetric(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, DataSource datasource, HeadlessSchema schema, Set<String> dimensions, List<String> queryDimensions, DataSource datasource, SemanticSchema schema, Set<String> dimensions,
Set<String> metrics) { Set<String> metrics) {
for (String field : fields) { for (String field : fields) {
if (queryDimensions.contains(field) || queryMetrics.contains(field)) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) {
@@ -229,7 +229,7 @@ public class SourceRender extends Renderer {
} }
} }
private static void addField(String field, String oriField, DataSource datasource, HeadlessSchema schema, private static void addField(String field, String oriField, DataSource datasource, SemanticSchema schema,
Set<String> dimensions, Set<String> dimensions,
Set<String> metrics) { Set<String> metrics) {
Optional<Dimension> dimension = datasource.getDimensions().stream() Optional<Dimension> dimension = datasource.getDimensions().stream()
@@ -272,7 +272,7 @@ public class SourceRender extends Renderer {
} }
} }
public static boolean isDimension(String name, DataSource datasource, HeadlessSchema schema) { public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) {
Optional<Dimension> dimension = datasource.getDimensions().stream() Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dimension.isPresent()) { if (dimension.isPresent()) {
@@ -317,7 +317,7 @@ public class SourceRender extends Renderer {
} }
public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryReq.getWhere(); String queryWhere = metricQueryReq.getWhere();
Set<String> whereFields = new HashSet<>(); Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>(); List<String> fieldWhere = new ArrayList<>();

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -58,7 +58,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getHeadlessModel().getDataDate()); queryStatement.getSemanticModel().getDataDate());
log.info("in generateSqlCommand, complete where:{}", where); log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);
@@ -109,7 +109,7 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setSql(parseSqlReq.getSql());
@@ -145,7 +145,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setAlias(metricTableName); metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getHeadlessModel().getDataDate()); String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getSemanticModel().getDataDate());
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);

13
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -28,21 +28,20 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) { public void convert(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
List<DimensionResp> dimensionResps = queryStatement.getHeadlessModel().getModelSchemaResps() List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream()
.stream().flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream()) .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.filter(dimSchemaResp -> !CollectionUtils.isEmpty(dimSchemaResp.getDefaultValues()))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensionResps)) { if (CollectionUtils.isEmpty(dimensions)) {
return; return;
} }
log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd); log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryStructCmd);
//add dimension default value to filter //add dimension default value to filter
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream() List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList()); .map(Filter::getBizName).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionFilterBizName)) { if (!CollectionUtils.isEmpty(dimensionFilterBizName)) {
return; return;
} }
for (DimensionResp dimensionResp : dimensionResps) { for (Dimension dimensionResp : dimensions) {
Filter filter = new Filter(); Filter filter = new Filter();
filter.setBizName(dimensionResp.getBizName()); filter.setBizName(dimensionResp.getBizName());
filter.setValue(dimensionResp.getDefaultValues()); filter.setValue(dimensionResp.getDefaultValues());

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
View File

@@ -48,7 +48,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement); MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq,
queryStatement.getHeadlessModel().getDataDate())); queryStatement.getSemanticModel().getDataDate()));
BeanUtils.copyProperties(metricReq, metricQueryReq); BeanUtils.copyProperties(metricReq, metricQueryReq);
} }
@@ -57,7 +57,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
metricQueryReq.setMetrics(queryStructReq.getMetrics()); metricQueryReq.setMetrics(queryStructReq.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups()); metricQueryReq.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getHeadlessModel().getDataDate()); queryStatement.getSemanticModel().getDataDate());
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where); metricQueryReq.setWhere(where);
@@ -71,7 +71,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
// support detail query // support detail query
if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) { if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) {
Map<Long, DataSource> dataSourceMap = queryStatement.getHeadlessModel().getModelMap(); Map<Long, DataSource> dataSourceMap = queryStatement.getSemanticModel().getModelMap();
for (Long modelId : queryStructReq.getModelIds()) { for (Long modelId : queryStructReq.getModelIds()) {
String modelBizName = dataSourceMap.get(modelId).getName(); String modelBizName = dataSourceMap.get(modelId).getName();
String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -30,7 +30,7 @@ public class QueryStatement {
private String viewSimplifySql = ""; private String viewSimplifySql = "";
private HeadlessModel headlessModel; private SemanticModel semanticModel;
public boolean isOk() { public boolean isOk() {
this.ok = "".equals(errMsg) && !"".equals(sql); this.ok = "".equals(errMsg) && !"".equals(sql);

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.pojo.yaml;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import java.util.List;
import lombok.Data; import lombok.Data;
@@ -10,6 +11,8 @@ public class DimensionYamlTpl {
private String name; private String name;
private String bizName;
private String owners; private String owners;
private String type; private String type;
@@ -19,4 +22,6 @@ public class DimensionYamlTpl {
private DimensionTimeTypeParamsTpl typeParams; private DimensionTimeTypeParamsTpl typeParams;
private DataTypeEnums dataType; private DataTypeEnums dataType;
private List<String> defaultValues;
} }

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java
View File

@@ -11,14 +11,14 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
@@ -55,21 +55,21 @@ import java.util.stream.Collectors;
public class HeadlessSchemaManager { public class HeadlessSchemaManager {
@Autowired @Autowired
private LoadingCache<String, HeadlessModel> loadingCache; private LoadingCache<String, SemanticModel> loadingCache;
private final Catalog catalog; private final Catalog catalog;
public HeadlessSchemaManager(Catalog catalog) { public HeadlessSchemaManager(Catalog catalog) {
this.catalog = catalog; this.catalog = catalog;
} }
public HeadlessModel reload(String rootPath) { public SemanticModel reload(String rootPath) {
HeadlessModel headlessModel = new HeadlessModel(); SemanticModel semanticModel = new SemanticModel();
headlessModel.setRootPath(rootPath); semanticModel.setRootPath(rootPath);
Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim())) Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
.collect(Collectors.toSet()); .collect(Collectors.toSet());
if (modelIds.isEmpty()) { if (modelIds.isEmpty()) {
log.error("get modelIds empty {}", rootPath); log.error("get modelIds empty {}", rootPath);
return headlessModel; return semanticModel;
} }
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>(); Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>(); List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
@@ -77,33 +77,33 @@ public class HeadlessSchemaManager {
Map<Long, String> modelIdName = new HashMap<>(); Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next()); DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
headlessModel.setDatabaseResp(databaseResp); semanticModel.setDatabaseResp(databaseResp);
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) { if (!CollectionUtils.isEmpty(modelRelas)) {
headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
} }
if (!dataModelYamlTpls.isEmpty()) { if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource) Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
headlessModel.setDatasourceMap(dataSourceMap); semanticModel.setDatasourceMap(dataSourceMap);
} }
if (!dimensionYamlTpls.isEmpty()) { if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>(); Map<String, List<Dimension>> dimensionMap = new HashMap<>();
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) { for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue())); dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
} }
headlessModel.setDimensionMap(dimensionMap); semanticModel.setDimensionMap(dimensionMap);
} }
if (!metricYamlTpls.isEmpty()) { if (!metricYamlTpls.isEmpty()) {
headlessModel.setMetrics(getMetrics(metricYamlTpls)); semanticModel.setMetrics(getMetrics(metricYamlTpls));
} }
return headlessModel; return semanticModel;
} }
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>(); //private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
public HeadlessModel get(String rootPath) throws Exception { public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath); rootPath = formatKey(rootPath);
HeadlessModel schema = loadingCache.get(rootPath); SemanticModel schema = loadingCache.get(rootPath);
if (schema == null) { if (schema == null) {
return null; return null;
} }
@@ -182,6 +182,8 @@ public class HeadlessSchemaManager {
dimension.setExpr(dimensionYamlTpl.getExpr()); dimension.setExpr(dimensionYamlTpl.getExpr());
dimension.setName(dimensionYamlTpl.getName()); dimension.setName(dimensionYamlTpl.getName());
dimension.setOwners(dimensionYamlTpl.getOwners()); dimension.setOwners(dimensionYamlTpl.getOwners());
dimension.setBizName(dimensionYamlTpl.getBizName());
dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
if (Objects.nonNull(dimensionYamlTpl.getDataType())) { if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType())); dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
} }
@@ -235,13 +237,13 @@ public class HeadlessSchemaManager {
return joinRelations; return joinRelations;
} }
public static void update(HeadlessSchema schema, List<Metric> metric) throws Exception { public static void update(SemanticSchema schema, List<Metric> metric) throws Exception {
if (schema != null) { if (schema != null) {
updateMetric(metric, schema.getMetrics()); updateMetric(metric, schema.getMetrics());
} }
} }
public static void update(HeadlessSchema schema, DataSource datasourceYamlTpl) throws Exception { public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception {
if (schema != null) { if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName(); String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream() Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
@@ -254,7 +256,7 @@ public class HeadlessSchemaManager {
} }
} }
public static void update(HeadlessSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls) public static void update(SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
throws Exception { throws Exception {
if (schema != null) { if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet() Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet()
@@ -320,15 +322,15 @@ public class HeadlessSchemaManager {
private Integer maximumSize = 1000; private Integer maximumSize = 1000;
@Bean @Bean
public LoadingCache<String, HeadlessModel> getCache() { public LoadingCache<String, SemanticModel> getCache() {
LoadingCache<String, HeadlessModel> cache LoadingCache<String, SemanticModel> cache
= CacheBuilder.newBuilder() = CacheBuilder.newBuilder()
.expireAfterWrite(saveMinutes, TimeUnit.MINUTES) .expireAfterWrite(saveMinutes, TimeUnit.MINUTES)
.initialCapacity(10) .initialCapacity(10)
.maximumSize(maximumSize).build( .maximumSize(maximumSize).build(
new CacheLoader<String, HeadlessModel>() { new CacheLoader<String, SemanticModel>() {
@Override @Override
public HeadlessModel load(String key) { public SemanticModel load(String key) {
log.info("load SemanticSchema [{}]", key); log.info("load SemanticSchema [{}]", key);
return HeadlessSchemaManager.this.reload(key); return HeadlessSchemaManager.this.reload(key);
} }

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java
View File

@@ -4,39 +4,34 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser; import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.List;
@Slf4j @Slf4j
@Component @Component
public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private final QueryParser queryParser; private final QueryParser queryParser;
private final Catalog catalog;
private final QueryUtils queryUtils; private final QueryUtils queryUtils;
private final QueryStructUtils queryStructUtils; private final QueryStructUtils queryStructUtils;
private final HeadlessSchemaManager headlessSchemaManager; private final HeadlessSchemaManager headlessSchemaManager;
public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog, public HeadlessQueryEngineImpl(QueryParser queryParser,
QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager, QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
QueryStructUtils queryStructUtils) { QueryStructUtils queryStructUtils) {
this.queryParser = queryParser; this.queryParser = queryParser;
this.catalog = catalog;
this.queryUtils = queryUtils; this.queryUtils = queryUtils;
this.headlessSchemaManager = headlessSchemaManager; this.headlessSchemaManager = headlessSchemaManager;
this.queryStructUtils = queryStructUtils; this.queryStructUtils = queryStructUtils;
@@ -57,7 +52,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public QueryStatement plan(QueryStatement queryStatement) throws Exception { public QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setHeadlessModel(getHeadLessModel(queryStatement)); queryStatement.setSemanticModel(getSemanticModel(queryStatement));
queryStatement = queryParser.logicSql(queryStatement); queryStatement = queryParser.logicSql(queryStatement);
queryUtils.checkSqlParse(queryStatement); queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
@@ -82,31 +77,31 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
} }
@Override @Override
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) { public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setParseSqlReq(sqlCommend); queryStatement.setParseSqlReq(sqlCommend);
queryStatement.setSql(sqlCommend.getSql()); queryStatement.setSql(sqlCommend.getSql());
queryStatement.setIsS2SQL(true); queryStatement.setIsS2SQL(true);
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement)); return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
} }
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) { public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setMetricReq(metricCommand); queryStatement.setMetricReq(metricCommand);
queryStatement.setIsS2SQL(false); queryStatement.setIsS2SQL(false);
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
return queryParser.parser(queryStatement); return queryParser.parser(queryStatement);
} }
private HeadlessModel getHeadLessModel(QueryStatement queryStatement) throws Exception { private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
HeadlessModel headlessModel = headlessSchemaManager.get(queryStructReq.getModelIdStr()); SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq); ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq);
headlessModel.setDataDate(itemDateResp); semanticModel.setDataDate(itemDateResp);
List<ModelSchemaResp> modelSchemaResps = catalog.getModelSchema(queryStructReq.getModelIds()); return semanticModel;
headlessModel.setModelSchemaResps(modelSchemaResps);
return headlessModel;
} }
} }

28
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.response.SqlParserResp; import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
@@ -26,20 +26,20 @@ import java.util.Map;
@Slf4j @Slf4j
class HeadlessParserServiceTest { class HeadlessParserServiceTest {
private static Map<String, HeadlessSchema> headlessSchemaMap = new HashMap<>(); private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>();
public static SqlParserResp parser(HeadlessSchema headlessSchema, MetricQueryReq metricCommand, boolean isAgg) { public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricCommand, boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp(); SqlParserResp sqlParser = new SqlParserResp();
if (metricCommand.getRootPath().isEmpty()) { if (metricCommand.getRootPath().isEmpty()) {
sqlParser.setErrMsg("rootPath empty"); sqlParser.setErrMsg("rootPath empty");
return sqlParser; return sqlParser;
} }
try { try {
if (headlessSchema == null) { if (semanticSchema == null) {
sqlParser.setErrMsg("headlessSchema not found"); sqlParser.setErrMsg("headlessSchema not found");
return sqlParser; return sqlParser;
} }
AggPlanner aggBuilder = new AggPlanner(headlessSchema); AggPlanner aggBuilder = new AggPlanner(semanticSchema);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricReq(metricCommand); queryStatement.setMetricReq(metricCommand);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
@@ -122,9 +122,9 @@ class HeadlessParserServiceTest {
identifies.add(identify); identifies.add(identify);
datasource.setIdentifiers(identifies); datasource.setIdentifiers(identifies);
HeadlessSchema headlessSchema = HeadlessSchema.newBuilder("s2").build(); SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getDatasource(datasource)); HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl(); DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page"); dimension1.setExpr("page");
@@ -133,7 +133,7 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>(); List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1); dimensionYamlTpls.add(dimension1);
HeadlessSchemaManager.update(headlessSchema, "s2_pv_uv_statis", HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
MetricYamlTpl metric1 = new MetricYamlTpl(); MetricYamlTpl metric1 = new MetricYamlTpl();
@@ -174,9 +174,9 @@ class HeadlessParserServiceTest {
List<ColumnOrder> orders = new ArrayList<>(); List<ColumnOrder> orders = new ArrayList<>();
orders.add(ColumnOrder.buildDesc("sys_imp_date")); orders.add(ColumnOrder.buildDesc("sys_imp_date"));
metricCommand.setOrder(orders); metricCommand.setOrder(orders);
System.out.println(parser(headlessSchema, metricCommand, true)); System.out.println(parser(semanticSchema, metricCommand, true));
addDepartment(headlessSchema); addDepartment(semanticSchema);
MetricQueryReq metricCommand2 = new MetricQueryReq(); MetricQueryReq metricCommand2 = new MetricQueryReq();
metricCommand2.setRootPath("s2"); metricCommand2.setRootPath("s2");
@@ -189,12 +189,12 @@ class HeadlessParserServiceTest {
List<ColumnOrder> orders2 = new ArrayList<>(); List<ColumnOrder> orders2 = new ArrayList<>();
orders2.add(ColumnOrder.buildDesc("sys_imp_date")); orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
metricCommand2.setOrder(orders2); metricCommand2.setOrder(orders2);
System.out.println(parser(headlessSchema, metricCommand2, true)); System.out.println(parser(semanticSchema, metricCommand2, true));
} }
private static void addDepartment(HeadlessSchema headlessSchema) { private static void addDepartment(SemanticSchema semanticSchema) {
DataModelYamlTpl datasource = new DataModelYamlTpl(); DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("user_department"); datasource.setName("user_department");
datasource.setSourceId(1L); datasource.setSourceId(1L);
@@ -240,7 +240,7 @@ class HeadlessParserServiceTest {
identifies.add(identify); identifies.add(identify);
datasource.setIdentifiers(identifies); datasource.setIdentifiers(identifies);
headlessSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource)); semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl(); DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department"); dimension1.setExpr("department");
@@ -249,7 +249,7 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>(); List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1); dimensionYamlTpls.add(dimension1);
headlessSchema.getDimension() semanticSchema.getDimension()
.put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); .put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
} }
} }