diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 9a75bbccf..0dc5aebbb 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -45,14 +45,6 @@ public class SemanticSchemaResp { .orElse(null); } - public List getMetrics(List bizNames) { - Map metricLowerToNameMap = metrics.stream().collect( - Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); - return bizNames.stream().map(String::toLowerCase) - .filter(entry -> metricLowerToNameMap.containsKey(entry)) - .map(entry -> metricLowerToNameMap.get(entry)).collect(Collectors.toList()); - } - public DimSchemaResp getDimension(String bizName) { return dimensions.stream() .filter(dimension -> bizName.equalsIgnoreCase(dimension.getBizName())).findFirst() @@ -64,14 +56,6 @@ public class SemanticSchemaResp { .orElse(null); } - public List getDimensions(List bizNames) { - Map dimLowerToNameMap = dimensions.stream().collect( - Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); - return bizNames.stream().map(String::toLowerCase) - .filter(entry -> dimLowerToNameMap.containsKey(entry)) - .map(entry -> dimLowerToNameMap.get(entry)).collect(Collectors.toList()); - } - public Set getNameFromBizNames(Set bizNames) { Set names = new HashSet<>(); for (String bizName : bizNames) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java index 7b8d8c772..f678cbb3d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java @@ -5,15 +5,21 @@ import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import lombok.Data; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +/** + * An ontology query comprises metrics/dimensions that are relevant to the semantic query. Note that + * metrics/dimensions in the ontology query must be a subset of an ontology. + */ @Data public class OntologyQuery { + private Set models = Sets.newHashSet(); private Set metrics = Sets.newHashSet(); private Set dimensions = Sets.newHashSet(); private Set fields = Sets.newHashSet(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index e782fb4ed..537b32a70 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.core.translator.parser; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; @@ -9,7 +11,9 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; @@ -18,9 +22,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; /** * This parser rewrites S2SQL including conversion from metric/dimension name to bizName and build @@ -57,16 +60,23 @@ public class SqlQueryParser implements QueryParser { } // build ontologyQuery + Ontology ontology = queryStatement.getOntology(); List allQueryFields = SqlSelectHelper.getAllSelectFields(sqlQuery.getSql()); - List queryMetrics = semanticSchema.getMetrics(allQueryFields); - List queryDimensions = semanticSchema.getDimensions(allQueryFields); OntologyQuery ontologyQuery = new OntologyQuery(); + queryStatement.setOntologyQuery(ontologyQuery); + + List queryMetrics = findQueryMetrics(ontology, allQueryFields); ontologyQuery.getMetrics().addAll(queryMetrics); + + List queryDimensions = findQueryDimensions(ontology, allQueryFields); ontologyQuery.getDimensions().addAll(queryDimensions); + List queryModels = findQueryModels(ontology, queryMetrics, queryDimensions); + ontologyQuery.getModels().addAll(queryModels); + AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), queryMetrics); ontologyQuery.setAggOption(sqlQueryAggOption); - queryStatement.setOntologyQuery(ontologyQuery); + log.info("parse sqlQuery [{}] ", sqlQuery); } @@ -126,4 +136,57 @@ public class SqlQueryParser implements QueryParser { queryStatement.getSqlQuery().setSql(newSql); } + public List findQueryMetrics(Ontology ontology, List bizNames) { + Map metricLowerToNameMap = ontology.getMetrics().stream().collect( + Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); + return bizNames.stream().map(String::toLowerCase) + .filter(entry -> metricLowerToNameMap.containsKey(entry)) + .map(entry -> metricLowerToNameMap.get(entry)).collect(Collectors.toList()); + } + + public List findQueryDimensions(Ontology ontology, List bizNames) { + Map dimLowerToNameMap = ontology.getDimensions().stream().collect( + Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); + return bizNames.stream().map(String::toLowerCase) + .filter(entry -> dimLowerToNameMap.containsKey(entry)) + .map(entry -> dimLowerToNameMap.get(entry)).collect(Collectors.toList()); + } + + public List findQueryModels(Ontology ontology, List queryMetrics, + List queryDimensions) { + // first, sort models based on the number of query metrics + Map modelMetricCount = Maps.newHashMap(); + queryMetrics.forEach(m -> { + if (!modelMetricCount.containsKey(m.getModelBizName())) { + modelMetricCount.put(m.getModelBizName(), 1); + } else { + int count = modelMetricCount.get(m.getModelBizName()); + modelMetricCount.put(m.getModelBizName(), count + 1); + } + }); + List metricsDataModels = modelMetricCount.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).map(e -> e.getKey()) + .collect(Collectors.toList()); + + // second, sort models based on the number of query dimensions + Map modelDimCount = Maps.newHashMap(); + queryDimensions.forEach(m -> { + if (!modelDimCount.containsKey(m.getModelBizName())) { + modelDimCount.put(m.getModelBizName(), 1); + } else { + int count = modelDimCount.get(m.getModelBizName()); + modelDimCount.put(m.getModelBizName(), count + 1); + } + }); + List dimDataModels = modelDimCount.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).map(e -> e.getKey()) + .collect(Collectors.toList()); + + Set dataModelNames = Sets.newLinkedHashSet(); + dataModelNames.addAll(dimDataModels); + dataModelNames.addAll(metricsDataModels); + return dataModelNames.stream().map(bizName -> ontology.getModelMap().get(bizName)) + .collect(Collectors.toList()); + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java index 1d95a5f51..720a9b1dc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.EngineType; @@ -127,56 +126,6 @@ public class DataModelNode extends SemanticNode { return sqlNode; } - public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery, - Set queryDimensions, Set queryMeasures) { - ontologyQuery.getMetrics().forEach(m -> { - if (Objects.nonNull(m.getMetricDefineByMeasureParams())) { - m.getMetricDefineByMeasureParams().getMeasures() - .forEach(mm -> queryMeasures.add(mm.getName())); - } - if (Objects.nonNull(m.getMetricDefineByFieldParams())) { - m.getMetricDefineByFieldParams().getFields() - .forEach(mm -> queryMeasures.add(mm.getFieldName())); - } - }); - } - - public static List getQueryDataModelsV2(Ontology ontology, OntologyQuery query) { - // first, sort models based on the number of query metrics - Map modelMetricCount = Maps.newHashMap(); - query.getMetrics().forEach(m -> { - if (!modelMetricCount.containsKey(m.getModelBizName())) { - modelMetricCount.put(m.getModelBizName(), 1); - } else { - int count = modelMetricCount.get(m.getModelBizName()); - modelMetricCount.put(m.getModelBizName(), count + 1); - } - }); - List metricsDataModels = modelMetricCount.entrySet().stream() - .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).map(e -> e.getKey()) - .collect(Collectors.toList()); - - // first, sort models based on the number of query dimensions - Map modelDimCount = Maps.newHashMap(); - query.getDimensions().forEach(m -> { - if (!modelDimCount.containsKey(m.getModelBizName())) { - modelDimCount.put(m.getModelBizName(), 1); - } else { - int count = modelDimCount.get(m.getModelBizName()); - modelDimCount.put(m.getModelBizName(), count + 1); - } - }); - List dimDataModels = modelDimCount.entrySet().stream() - .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).map(e -> e.getKey()) - .collect(Collectors.toList()); - - Set dataModelNames = Sets.newLinkedHashSet(); - dataModelNames.addAll(dimDataModels); - dataModelNames.addAll(metricsDataModels); - return dataModelNames.stream().map(bizName -> ontology.getModelMap().get(bizName)) - .collect(Collectors.toList()); - } - public static List getQueryDataModels(Ontology ontology, OntologyQuery ontologyQuery) { // get query measures and dimensions @@ -212,6 +161,20 @@ public class DataModelNode extends SemanticNode { return relatedDataModels; } + public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery, + Set queryDimensions, Set queryMeasures) { + ontologyQuery.getMetrics().forEach(m -> { + if (Objects.nonNull(m.getMetricDefineByMeasureParams())) { + m.getMetricDefineByMeasureParams().getMeasures() + .forEach(mm -> queryMeasures.add(mm.getName())); + } + if (Objects.nonNull(m.getMetricDefineByFieldParams())) { + m.getMetricDefineByFieldParams().getFields() + .forEach(mm -> queryMeasures.add(mm.getFieldName())); + } + }); + } + private static ModelResp findBaseModel(Ontology ontology, OntologyQuery query) { ModelResp dataModel = null; // first, try to find the model with the most query metrics diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java index 74a162041..1852d9703 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java @@ -41,14 +41,12 @@ public class SqlBuilder { ontologyQuery.setLimit(0L); } - // find relevant data models - List dataModels = - DataModelNode.getQueryDataModelsV2(schema.getOntology(), ontologyQuery); + Set dataModels = ontologyQuery.getModels(); if (dataModels == null || dataModels.isEmpty()) { throw new Exception("data model not found"); } - TableView tableView = render(ontologyQuery, dataModels, scope, schema); + TableView tableView = render(ontologyQuery, new ArrayList<>(dataModels), scope, schema); SqlNode parserNode = tableView.build(); DatabaseResp database = queryStatement.getOntology().getDatabase(); EngineType engineType = EngineType.fromString(database.getType());