diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index 6e3e8222d..46e93a496 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -30,7 +30,7 @@ public class JdbcExecutor implements QueryExecutor { return null; } log.info("query SQL: {}", queryStatement.getSql()); - DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); + DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp(); log.info("database info:{}", databaseResp); QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java index cacbdef2f..9ba1041d7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java @@ -128,7 +128,7 @@ public class QueryParser { tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); tableSql.setModelIds(queryStatement.getModelIds()); - tableSql.setHeadlessModel(queryStatement.getHeadlessModel()); + tableSql.setSemanticModel(queryStatement.getSemanticModel()); if (isSingleMetricTable) { tableSql.setViewSql(parseSqlReq.getSql()); tableSql.setViewAlias(metricTable.getAlias()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java index bd1cd2b41..a45dab304 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java @@ -4,8 +4,8 @@ import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import java.util.Objects; @@ -22,14 +22,14 @@ public class CalciteSqlParser implements SqlParser { @Override public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception { MetricQueryReq metricReq = queryStatement.getMetricReq(); - HeadlessModel headlessModel = queryStatement.getHeadlessModel(); - if (headlessModel == null) { + SemanticModel semanticModel = queryStatement.getSemanticModel(); + if (semanticModel == null) { queryStatement.setErrMsg("semanticSchema not found"); return queryStatement; } queryStatement.setMetricReq(metricReq); - HeadlessSchema headlessSchema = getSemanticSchema(headlessModel, queryStatement); - AggPlanner aggBuilder = new AggPlanner(headlessSchema); + SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); + AggPlanner aggBuilder = new AggPlanner(semanticSchema); aggBuilder.explain(queryStatement, isAgg); queryStatement.setSql(aggBuilder.getSql()); queryStatement.setSourceId(aggBuilder.getSourceId()); @@ -46,15 +46,15 @@ public class CalciteSqlParser implements SqlParser { return queryStatement; } - private HeadlessSchema getSemanticSchema(HeadlessModel headlessModel, QueryStatement queryStatement) { - HeadlessSchema headlessSchema = HeadlessSchema.newBuilder(headlessModel.getRootPath()).build(); - headlessSchema.setDatasource(headlessModel.getDatasourceMap()); - headlessSchema.setDimension(headlessModel.getDimensionMap()); - headlessSchema.setMetric(headlessModel.getMetrics()); - headlessSchema.setJoinRelations(headlessModel.getJoinRelations()); - headlessSchema.setRuntimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) + private SemanticSchema getSemanticSchema(SemanticModel semanticModel, QueryStatement queryStatement) { + SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build(); + semanticSchema.setDatasource(semanticModel.getDatasourceMap()); + semanticSchema.setDimension(semanticModel.getDimensionMap()); + semanticSchema.setMetric(semanticModel.getMetrics()); + semanticSchema.setJoinRelations(semanticModel.getJoinRelations()); + semanticSchema.setRuntimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) .enableOptimize(queryStatement.getEnableOptimize()).build()); - return headlessSchema; + return semanticSchema; } private String getSqlByView(String sql, String viewSql, String viewAlias) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java index 6cb87f483..69a1be47b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; @@ -34,7 +34,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; public class AggPlanner implements Planner { private MetricQueryReq metricReq; - private HeadlessSchema schema; + private SemanticSchema schema; private SqlValidatorScope scope; private Stack dataSets = new Stack<>(); private SqlNode parserNode; @@ -42,7 +42,7 @@ public class AggPlanner implements Planner { private boolean isAgg = false; private AggOption aggOption = AggOption.DEFAULT; - public AggPlanner(HeadlessSchema schema) { + public AggPlanner(SemanticSchema schema) { this.schema = schema; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java index 8a09d8aee..a9c9a1d95 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticItem; +import java.util.List; import lombok.Builder; import lombok.Data; @@ -15,8 +16,9 @@ public class Dimension implements SemanticItem { private String type; private String expr; private DimensionTimeTypeParams dimensionTimeTypeParams; - private DataType dataType = DataType.UNKNOWN; + private String bizName; + private List defaultValues; @Override public String getName() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java similarity index 88% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java index e86428d1b..c75456509 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java @@ -2,18 +2,16 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import lombok.Data; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.Data; @Data -public class HeadlessModel { +public class SemanticModel { private String rootPath; private List metrics = new ArrayList<>(); @@ -23,7 +21,6 @@ public class HeadlessModel { private List joinRelations; private ItemDateResp dataDate; private DatabaseResp databaseResp; - private List modelSchemaResps; public List getDimensions() { return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index 9c528dd51..940bdbe0c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -27,7 +27,7 @@ public class SchemaBuilder { public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1"; public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2"; - public static SqlValidatorScope getScope(HeadlessSchema schema) throws Exception { + public static SqlValidatorScope getScope(SemanticSchema schema) throws Exception { Map nameToTypeMap = new HashMap<>(); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); rootSchema.add(schema.getRootPath(), schema); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java similarity index 77% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSchema.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java index 62eaa2d5c..cd0486520 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/HeadlessSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSchema.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.SchemaVersion; import org.apache.calcite.schema.Table; @@ -17,19 +17,19 @@ import java.util.List; import java.util.Map; -public class HeadlessSchema extends AbstractSchema { +public class SemanticSchema extends AbstractSchema { private final String rootPath; private final Map tableMap; - private HeadlessModel headlessModel = new HeadlessModel(); + private SemanticModel semanticModel = new SemanticModel(); private List joinRelations; private RuntimeOptions runtimeOptions; - private HeadlessSchema(String rootPath, Map tableMap) { + private SemanticSchema(String rootPath, Map tableMap) { this.rootPath = rootPath; this.tableMap = tableMap; } @@ -42,12 +42,12 @@ public class HeadlessSchema extends AbstractSchema { return rootPath; } - public void setSemanticModel(HeadlessModel headlessModel) { - this.headlessModel = headlessModel; + public void setSemanticModel(SemanticModel semanticModel) { + this.semanticModel = semanticModel; } - public HeadlessModel getSemanticModel() { - return headlessModel; + public SemanticModel getSemanticModel() { + return semanticModel; } @Override @@ -61,35 +61,35 @@ public class HeadlessSchema extends AbstractSchema { } public Map getDatasource() { - return headlessModel.getDatasourceMap(); + return semanticModel.getDatasourceMap(); } public void setDatasource(Map datasource) { - headlessModel.setDatasourceMap(datasource); + semanticModel.setDatasourceMap(datasource); } public Map> getDimension() { - return headlessModel.getDimensionMap(); + return semanticModel.getDimensionMap(); } public void setDimension(Map> dimensions) { - headlessModel.setDimensionMap(dimensions); + semanticModel.setDimensionMap(dimensions); } public List getMetrics() { - return headlessModel.getMetrics(); + return semanticModel.getMetrics(); } public void setMetric(List metric) { - headlessModel.setMetrics(metric); + semanticModel.setMetrics(metric); } public void setMaterializationList(List materializationList) { - headlessModel.setMaterializationList(materializationList); + semanticModel.setMaterializationList(materializationList); } public List getMaterializationList() { - return headlessModel.getMaterializationList(); + return semanticModel.getMaterializationList(); } public void setJoinRelations(List joinRelations) { @@ -131,8 +131,8 @@ public class HeadlessSchema extends AbstractSchema { return this; } - public HeadlessSchema build() { - return new HeadlessSchema(rootPath, tableMap); + public SemanticSchema build() { + return new SemanticSchema(rootPath, tableMap); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Optimization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Optimization.java index ef5bfd323..a811d6034 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Optimization.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Optimization.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; public interface Optimization { - public void visit(HeadlessSchema headlessSchema); + public void visit(SemanticSchema semanticSchema); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java index ef42adc48..29bb62b9f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/Renderer.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import java.util.HashSet; import java.util.List; @@ -36,7 +36,7 @@ public abstract class Renderer { return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name)).findFirst(); } - public static Optional getMetricByName(String name, HeadlessSchema schema) { + public static Optional getMetricByName(String name, SemanticSchema schema) { Optional metric = schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(name)) .findFirst(); return metric; @@ -47,7 +47,7 @@ public abstract class Renderer { } public static MetricNode buildMetricNode(String metric, DataSource datasource, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg, String alias) throws Exception { + SemanticSchema schema, boolean nonAgg, String alias) throws Exception { Optional metricOpt = getMetricByName(metric, schema); MetricNode metricNode = new MetricNode(); if (metricOpt.isPresent()) { @@ -105,5 +105,5 @@ public abstract class Renderer { } public abstract void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception; + SemanticSchema schema, boolean nonAgg) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index 5ac0f4e5b..a815da7aa 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode; import java.util.ArrayList; @@ -149,7 +149,7 @@ public class DataSourceNode extends SemanticNode { return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_")); } - public static void getQueryDimensionMeasure(HeadlessSchema schema, MetricQueryReq metricCommand, + public static void getQueryDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand, Set queryDimension, List measures) { queryDimension.addAll(metricCommand.getDimensions().stream() .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d) @@ -161,7 +161,7 @@ public class DataSourceNode extends SemanticNode { } - public static void mergeQueryFilterDimensionMeasure(HeadlessSchema schema, MetricQueryReq metricCommand, + public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricQueryReq metricCommand, Set queryDimension, List measures, SqlValidatorScope scope) throws Exception { if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { @@ -184,7 +184,7 @@ public class DataSourceNode extends SemanticNode { } } - public static List getMatchDataSources(SqlValidatorScope scope, HeadlessSchema schema, + public static List getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema, MetricQueryReq metricCommand) throws Exception { List dataSources = new ArrayList<>(); @@ -283,7 +283,7 @@ public class DataSourceNode extends SemanticNode { } private static List getLinkDataSourcesByJoinRelation(Set queryDimension, List measures, - DataSource baseDataSource, HeadlessSchema schema) { + DataSource baseDataSource, SemanticSchema schema) { Set linkDataSourceName = new HashSet<>(); List linkDataSources = new ArrayList<>(); Set before = new HashSet<>(); @@ -370,7 +370,7 @@ public class DataSourceNode extends SemanticNode { Set queryDimension, List measures, DataSource baseDataSource, - HeadlessSchema schema) { + SemanticSchema schema) { Set linkDataSourceName = new HashSet<>(); List linkDataSources = new ArrayList<>(); for (Map.Entry entry : schema.getDatasource().entrySet()) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java index c071327c3..630bfd215 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule; import java.util.ArrayList; @@ -335,7 +335,7 @@ public abstract class SemanticNode { return parseInfo; } - public static SqlNode optimize(SqlValidatorScope scope, HeadlessSchema schema, SqlNode sqlNode) { + public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode) { try { HepProgramBuilder hepProgramBuilder = new HepProgramBuilder(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/optimizer/FilterToGroupScanRule.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/optimizer/FilterToGroupScanRule.java index 6efeed823..c598aa289 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/optimizer/FilterToGroupScanRule.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/optimizer/FilterToGroupScanRule.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import java.util.ArrayList; import java.util.Iterator; @@ -44,23 +44,23 @@ public class FilterToGroupScanRule extends RelRule }).as(FilterTableScanRule.Config.class); - private HeadlessSchema headlessSchema; + private SemanticSchema semanticSchema; - public FilterToGroupScanRule(FilterTableScanRule.Config config, HeadlessSchema headlessSchema) { + public FilterToGroupScanRule(FilterTableScanRule.Config config, SemanticSchema semanticSchema) { super(config); - this.headlessSchema = headlessSchema; + this.semanticSchema = semanticSchema; } public void onMatch(RelOptRuleCall call) { if (call.rels.length != 4) { return; } - if (Objects.isNull(headlessSchema.getRuntimeOptions()) || Objects.isNull( - headlessSchema.getRuntimeOptions().getMinMaxTime()) || headlessSchema.getRuntimeOptions() + if (Objects.isNull(semanticSchema.getRuntimeOptions()) || Objects.isNull( + semanticSchema.getRuntimeOptions().getMinMaxTime()) || semanticSchema.getRuntimeOptions() .getMinMaxTime().getLeft().isEmpty()) { return; } - Triple minMax = headlessSchema.getRuntimeOptions().getMinMaxTime(); + Triple minMax = semanticSchema.getRuntimeOptions().getMinMaxTime(); Filter filter = (Filter) call.rel(0); Project project0 = (Project) call.rel(1); Project project1 = (Project) call.rel(3); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java index 359ae8ba1..653085e03 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/FilterRender.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.FilterNode; @@ -30,7 +30,7 @@ public class FilterRender extends Renderer { @Override public void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { TableView tableView = super.tableView; SqlNode filterNode = null; List queryMetrics = new ArrayList<>(metricCommand.getMetrics()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java index fea5b5060..c77195258 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/JoinRender.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.AggFunctionNode; @@ -49,7 +49,7 @@ public class JoinRender extends Renderer { @Override public void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); //dataSources = getOrderSource(dataSources); Set whereFields = new HashSet<>(); @@ -141,7 +141,7 @@ public class JoinRender extends Renderer { private void doMetric(Map innerSelect, TableView filterView, List queryMetrics, List reqMetrics, DataSource dataSource, Set sourceMeasure, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); for (String m : reqMetrics) { if (getMatchMetric(schema, sourceMeasure, m, queryMetrics)) { @@ -175,7 +175,7 @@ public class JoinRender extends Renderer { private void doDimension(Map innerSelect, Set filterDimension, List queryDimension, List reqDimensions, DataSource dataSource, Set dimension, SqlValidatorScope scope, - HeadlessSchema schema) throws Exception { + SemanticSchema schema) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); for (String d : reqDimensions) { if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) { @@ -198,7 +198,7 @@ public class JoinRender extends Renderer { Collectors.toSet()); } - private boolean getMatchMetric(HeadlessSchema schema, Set sourceMeasure, String m, + private boolean getMatchMetric(SemanticSchema schema, Set sourceMeasure, String m, List queryMetrics) { Optional metric = schema.getMetrics().stream().filter(mm -> mm.getName().equalsIgnoreCase(m)) .findFirst(); @@ -219,7 +219,7 @@ public class JoinRender extends Renderer { return isAdd; } - private boolean getMatchDimension(HeadlessSchema schema, Set sourceDimension, DataSource dataSource, + private boolean getMatchDimension(SemanticSchema schema, Set sourceDimension, DataSource dataSource, String d, List queryDimension) { String oriDimension = d; boolean isAdd = false; @@ -254,7 +254,7 @@ public class JoinRender extends Renderer { private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map before, DataSource dataSource, - HeadlessSchema schema, SqlValidatorScope scope) + SemanticSchema schema, SqlValidatorScope scope) throws Exception { SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope); SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); @@ -289,7 +289,7 @@ public class JoinRender extends Renderer { ); } - private JoinRelation getMatchJoinRelation(Map before, TableView tableView, HeadlessSchema schema) { + private JoinRelation getMatchJoinRelation(Map before, TableView tableView, SemanticSchema schema) { JoinRelation matchJoinRelation = JoinRelation.builder().build(); if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { for (JoinRelation joinRelation : schema.getJoinRelations()) { @@ -333,7 +333,7 @@ public class JoinRender extends Renderer { return condition; } - private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, HeadlessSchema schema, + private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) throws Exception { Set selectLeft = SemanticNode.getSelect(left.getTable()); @@ -413,7 +413,7 @@ public class JoinRender extends Renderer { } } - private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, HeadlessSchema schema, + private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema, SqlValidatorScope scope) throws Exception { if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) && Materialization.TimePartType.ZIPPER.equals(right.getDataSource().getTimePartType())) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java index eccb0dd4e..860b09e75 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/OutputRender.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode; @@ -25,7 +25,7 @@ public class OutputRender extends Renderer { @Override public void render(MetricQueryReq metricCommand, List dataSources, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { TableView selectDataSet = super.tableView; for (String dimension : metricCommand.getDimensions()) { selectDataSet.getMeasure().add(SemanticNode.parse(dimension, scope)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java index ab5e325de..118822404 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode; @@ -41,7 +41,7 @@ public class SourceRender extends Renderer { public static TableView renderOne(String alias, List fieldWheres, List reqMetrics, List reqDimensions, String queryWhere, DataSource datasource, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { TableView dataSet = new TableView(); TableView output = new TableView(); @@ -93,7 +93,7 @@ public class SourceRender extends Renderer { return output; } - private static void buildDimension(String alias, String dimension, DataSource datasource, HeadlessSchema schema, + private static void buildDimension(String alias, String dimension, DataSource datasource, SemanticSchema schema, boolean nonAgg, Set extendFields, TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception { List dimensionList = schema.getDimension().get(datasource.getName()); @@ -161,7 +161,7 @@ public class SourceRender extends Renderer { private static List getWhereMeasure(List fields, List queryMetrics, List queryDimensions, Set extendFields, DataSource datasource, SqlValidatorScope scope, - HeadlessSchema schema, + SemanticSchema schema, boolean nonAgg) throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); @@ -205,7 +205,7 @@ public class SourceRender extends Renderer { private static void mergeWhere(List fields, TableView dataSet, TableView outputSet, List queryMetrics, List queryDimensions, Set extendFields, DataSource datasource, SqlValidatorScope scope, - HeadlessSchema schema, + SemanticSchema schema, boolean nonAgg) throws Exception { List whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, scope, schema, @@ -215,7 +215,7 @@ public class SourceRender extends Renderer { } public static void whereDimMetric(List fields, List queryMetrics, - List queryDimensions, DataSource datasource, HeadlessSchema schema, Set dimensions, + List queryDimensions, DataSource datasource, SemanticSchema schema, Set dimensions, Set metrics) { for (String field : fields) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) { @@ -229,7 +229,7 @@ public class SourceRender extends Renderer { } } - private static void addField(String field, String oriField, DataSource datasource, HeadlessSchema schema, + private static void addField(String field, String oriField, DataSource datasource, SemanticSchema schema, Set dimensions, Set metrics) { Optional dimension = datasource.getDimensions().stream() @@ -272,7 +272,7 @@ public class SourceRender extends Renderer { } } - public static boolean isDimension(String name, DataSource datasource, HeadlessSchema schema) { + public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) { Optional dimension = datasource.getDimensions().stream() .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); if (dimension.isPresent()) { @@ -317,7 +317,7 @@ public class SourceRender extends Renderer { } public void render(MetricQueryReq metricQueryReq, List dataSources, SqlValidatorScope scope, - HeadlessSchema schema, boolean nonAgg) throws Exception { + SemanticSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricQueryReq.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index f8eb19502..4554441ac 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -58,7 +58,7 @@ public class CalculateAggConverter implements HeadlessConverter { metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setDimensions(queryStructReq.getGroups()); String where = sqlGenerateUtils.generateWhere(queryStructReq, - queryStatement.getHeadlessModel().getDataDate()); + queryStatement.getSemanticModel().getDataDate()); log.info("in generateSqlCommand, complete where:{}", where); metricTable.setWhere(where); metricTable.setAggOption(AggOption.AGGREGATION); @@ -109,7 +109,7 @@ public class CalculateAggConverter implements HeadlessConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); - DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); + DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp(); ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); sqlCommend.setSql(parseSqlReq.getSql()); @@ -145,7 +145,7 @@ public class CalculateAggConverter implements HeadlessConverter { metricTable.setAlias(metricTableName); metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setDimensions(queryStructReq.getGroups()); - String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getHeadlessModel().getDataDate()); + String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getSemanticModel().getDataDate()); log.info("in generateSqlCommend, complete where:{}", where); metricTable.setWhere(where); metricTable.setAggOption(AggOption.AGGREGATION); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java index 3a5afd1d5..4b3247498 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.parser.converter; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.core.parser.HeadlessConverter; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -28,21 +28,20 @@ public class DefaultDimValueConverter implements HeadlessConverter { @Override public void convert(QueryStatement queryStatement) { QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - List dimensionResps = queryStatement.getHeadlessModel().getModelSchemaResps() - .stream().flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream()) - .filter(dimSchemaResp -> !CollectionUtils.isEmpty(dimSchemaResp.getDefaultValues())) + List dimensions = queryStatement.getSemanticModel().getDimensions().stream() + .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(dimensionResps)) { + if (CollectionUtils.isEmpty(dimensions)) { return; } - log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd); + log.info("dimension with default values:{}, queryStruct:{}", dimensions, queryStructCmd); //add dimension default value to filter List dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream() .map(Filter::getBizName).collect(Collectors.toList()); if (!CollectionUtils.isEmpty(dimensionFilterBizName)) { return; } - for (DimensionResp dimensionResp : dimensionResps) { + for (Dimension dimensionResp : dimensions) { Filter filter = new Filter(); filter.setBizName(dimensionResp.getBizName()); filter.setValue(dimensionResp.getDefaultValues()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java index 0b4b7db64..9e9da892f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java @@ -48,7 +48,7 @@ public class ParserDefaultConverter implements HeadlessConverter { MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement); queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, - queryStatement.getHeadlessModel().getDataDate())); + queryStatement.getSemanticModel().getDataDate())); BeanUtils.copyProperties(metricReq, metricQueryReq); } @@ -57,7 +57,7 @@ public class ParserDefaultConverter implements HeadlessConverter { metricQueryReq.setMetrics(queryStructReq.getMetrics()); metricQueryReq.setDimensions(queryStructReq.getGroups()); String where = sqlGenerateUtils.generateWhere(queryStructReq, - queryStatement.getHeadlessModel().getDataDate()); + queryStatement.getSemanticModel().getDataDate()); log.info("in generateSqlCommend, complete where:{}", where); metricQueryReq.setWhere(where); @@ -71,7 +71,7 @@ public class ParserDefaultConverter implements HeadlessConverter { // support detail query if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) { - Map dataSourceMap = queryStatement.getHeadlessModel().getModelMap(); + Map dataSourceMap = queryStatement.getSemanticModel().getModelMap(); for (Long modelId : queryStructReq.getModelIds()) { String modelBizName = dataSourceMap.get(modelId).getName(); String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 3b87aa157..a41e6d378 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import java.util.List; import lombok.Data; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -30,7 +30,7 @@ public class QueryStatement { private String viewSimplifySql = ""; - private HeadlessModel headlessModel; + private SemanticModel semanticModel; public boolean isOk() { this.ok = "".equals(errMsg) && !"".equals(sql); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java index f8fc8937a..0ff5d1e6a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.pojo.yaml; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; +import java.util.List; import lombok.Data; @@ -10,6 +11,8 @@ public class DimensionYamlTpl { private String name; + private String bizName; + private String owners; private String type; @@ -19,4 +22,6 @@ public class DimensionYamlTpl { private DimensionTimeTypeParamsTpl typeParams; private DataTypeEnums dataType; + + private List defaultValues; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java index 29c3b36e5..0d37669fe 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java @@ -11,14 +11,14 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; @@ -55,21 +55,21 @@ import java.util.stream.Collectors; public class HeadlessSchemaManager { @Autowired - private LoadingCache loadingCache; + private LoadingCache loadingCache; private final Catalog catalog; public HeadlessSchemaManager(Catalog catalog) { this.catalog = catalog; } - public HeadlessModel reload(String rootPath) { - HeadlessModel headlessModel = new HeadlessModel(); - headlessModel.setRootPath(rootPath); + public SemanticModel reload(String rootPath) { + SemanticModel semanticModel = new SemanticModel(); + semanticModel.setRootPath(rootPath); Set modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim())) .collect(Collectors.toSet()); if (modelIds.isEmpty()) { log.error("get modelIds empty {}", rootPath); - return headlessModel; + return semanticModel; } Map> dimensionYamlTpls = new HashMap<>(); List dataModelYamlTpls = new ArrayList<>(); @@ -77,33 +77,33 @@ public class HeadlessSchemaManager { Map modelIdName = new HashMap<>(); catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next()); - headlessModel.setDatabaseResp(databaseResp); + semanticModel.setDatabaseResp(databaseResp); List modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); if (!CollectionUtils.isEmpty(modelRelas)) { - headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); + semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { Map dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource) .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); - headlessModel.setDatasourceMap(dataSourceMap); + semanticModel.setDatasourceMap(dataSourceMap); } if (!dimensionYamlTpls.isEmpty()) { Map> dimensionMap = new HashMap<>(); for (Map.Entry> entry : dimensionYamlTpls.entrySet()) { dimensionMap.put(entry.getKey(), getDimensions(entry.getValue())); } - headlessModel.setDimensionMap(dimensionMap); + semanticModel.setDimensionMap(dimensionMap); } if (!metricYamlTpls.isEmpty()) { - headlessModel.setMetrics(getMetrics(metricYamlTpls)); + semanticModel.setMetrics(getMetrics(metricYamlTpls)); } - return headlessModel; + return semanticModel; } //private Map semanticSchemaMap = new HashMap<>(); - public HeadlessModel get(String rootPath) throws Exception { + public SemanticModel get(String rootPath) throws Exception { rootPath = formatKey(rootPath); - HeadlessModel schema = loadingCache.get(rootPath); + SemanticModel schema = loadingCache.get(rootPath); if (schema == null) { return null; } @@ -182,6 +182,8 @@ public class HeadlessSchemaManager { dimension.setExpr(dimensionYamlTpl.getExpr()); dimension.setName(dimensionYamlTpl.getName()); dimension.setOwners(dimensionYamlTpl.getOwners()); + dimension.setBizName(dimensionYamlTpl.getBizName()); + dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues()); if (Objects.nonNull(dimensionYamlTpl.getDataType())) { dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType())); } @@ -235,13 +237,13 @@ public class HeadlessSchemaManager { return joinRelations; } - public static void update(HeadlessSchema schema, List metric) throws Exception { + public static void update(SemanticSchema schema, List metric) throws Exception { if (schema != null) { updateMetric(metric, schema.getMetrics()); } } - public static void update(HeadlessSchema schema, DataSource datasourceYamlTpl) throws Exception { + public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception { if (schema != null) { String dataSourceName = datasourceYamlTpl.getName(); Optional> datasourceYamlTplMap = schema.getDatasource().entrySet().stream() @@ -254,7 +256,7 @@ public class HeadlessSchemaManager { } } - public static void update(HeadlessSchema schema, String datasourceBizName, List dimensionYamlTpls) + public static void update(SemanticSchema schema, String datasourceBizName, List dimensionYamlTpls) throws Exception { if (schema != null) { Optional>> datasourceYamlTplMap = schema.getDimension().entrySet() @@ -320,15 +322,15 @@ public class HeadlessSchemaManager { private Integer maximumSize = 1000; @Bean - public LoadingCache getCache() { - LoadingCache cache + public LoadingCache getCache() { + LoadingCache cache = CacheBuilder.newBuilder() .expireAfterWrite(saveMinutes, TimeUnit.MINUTES) .initialCapacity(10) .maximumSize(maximumSize).build( - new CacheLoader() { + new CacheLoader() { @Override - public HeadlessModel load(String key) { + public SemanticModel load(String key) { log.info("load SemanticSchema [{}]", key); return HeadlessSchemaManager.this.reload(key); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java index 59875615c..f872f6c29 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java @@ -4,39 +4,34 @@ import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.QueryParser; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; -import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.List; @Slf4j @Component public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { private final QueryParser queryParser; - private final Catalog catalog; private final QueryUtils queryUtils; private final QueryStructUtils queryStructUtils; private final HeadlessSchemaManager headlessSchemaManager; - public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog, + public HeadlessQueryEngineImpl(QueryParser queryParser, QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager, QueryStructUtils queryStructUtils) { this.queryParser = queryParser; - this.catalog = catalog; this.queryUtils = queryUtils; this.headlessSchemaManager = headlessSchemaManager; this.queryStructUtils = queryStructUtils; @@ -57,7 +52,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { public QueryStatement plan(QueryStatement queryStatement) throws Exception { queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setHeadlessModel(getHeadLessModel(queryStatement)); + queryStatement.setSemanticModel(getSemanticModel(queryStatement)); queryStatement = queryParser.logicSql(queryStatement); queryUtils.checkSqlParse(queryStatement); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); @@ -82,31 +77,31 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { } @Override - public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception { QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setParseSqlReq(sqlCommend); queryStatement.setSql(sqlCommend.getSql()); queryStatement.setIsS2SQL(true); + queryStatement.setSemanticModel(getSemanticModel(queryStatement)); return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement)); } - public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception { QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setMetricReq(metricCommand); queryStatement.setIsS2SQL(false); + queryStatement.setSemanticModel(getSemanticModel(queryStatement)); return queryParser.parser(queryStatement); } - private HeadlessModel getHeadLessModel(QueryStatement queryStatement) throws Exception { + private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception { QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); - HeadlessModel headlessModel = headlessSchemaManager.get(queryStructReq.getModelIdStr()); + SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr()); ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq); - headlessModel.setDataDate(itemDateResp); - List modelSchemaResps = catalog.getModelSchema(queryStructReq.getModelIds()); - headlessModel.setModelSchemaResps(modelSchemaResps); - return headlessModel; + semanticModel.setDataDate(itemDateResp); + return semanticModel; } } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index f112d396b..826fd723d 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.response.SqlParserResp; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; @@ -26,20 +26,20 @@ import java.util.Map; @Slf4j class HeadlessParserServiceTest { - private static Map headlessSchemaMap = new HashMap<>(); + private static Map headlessSchemaMap = new HashMap<>(); - public static SqlParserResp parser(HeadlessSchema headlessSchema, MetricQueryReq metricCommand, boolean isAgg) { + public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricCommand, boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); if (metricCommand.getRootPath().isEmpty()) { sqlParser.setErrMsg("rootPath empty"); return sqlParser; } try { - if (headlessSchema == null) { + if (semanticSchema == null) { sqlParser.setErrMsg("headlessSchema not found"); return sqlParser; } - AggPlanner aggBuilder = new AggPlanner(headlessSchema); + AggPlanner aggBuilder = new AggPlanner(semanticSchema); QueryStatement queryStatement = new QueryStatement(); queryStatement.setMetricReq(metricCommand); aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg)); @@ -122,9 +122,9 @@ class HeadlessParserServiceTest { identifies.add(identify); datasource.setIdentifiers(identifies); - HeadlessSchema headlessSchema = HeadlessSchema.newBuilder("s2").build(); + SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build(); - HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getDatasource(datasource)); + HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("page"); @@ -133,7 +133,7 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - HeadlessSchemaManager.update(headlessSchema, "s2_pv_uv_statis", + HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis", HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); MetricYamlTpl metric1 = new MetricYamlTpl(); @@ -174,9 +174,9 @@ class HeadlessParserServiceTest { List orders = new ArrayList<>(); orders.add(ColumnOrder.buildDesc("sys_imp_date")); metricCommand.setOrder(orders); - System.out.println(parser(headlessSchema, metricCommand, true)); + System.out.println(parser(semanticSchema, metricCommand, true)); - addDepartment(headlessSchema); + addDepartment(semanticSchema); MetricQueryReq metricCommand2 = new MetricQueryReq(); metricCommand2.setRootPath("s2"); @@ -189,12 +189,12 @@ class HeadlessParserServiceTest { List orders2 = new ArrayList<>(); orders2.add(ColumnOrder.buildDesc("sys_imp_date")); metricCommand2.setOrder(orders2); - System.out.println(parser(headlessSchema, metricCommand2, true)); + System.out.println(parser(semanticSchema, metricCommand2, true)); } - private static void addDepartment(HeadlessSchema headlessSchema) { + private static void addDepartment(SemanticSchema semanticSchema) { DataModelYamlTpl datasource = new DataModelYamlTpl(); datasource.setName("user_department"); datasource.setSourceId(1L); @@ -240,7 +240,7 @@ class HeadlessParserServiceTest { identifies.add(identify); datasource.setIdentifiers(identifies); - headlessSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource)); + semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("department"); @@ -249,7 +249,7 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - headlessSchema.getDimension() + semanticSchema.getDimension() .put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); } }