topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless translator.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-26 15:21:57 +08:00
parent b84dde3799
commit 40bfcdce2c
14 changed files with 65 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java
View File

@@ -1,16 +1,17 @@
package com.tencent.supersonic.headless.core.translator.calcite.s2sql; package com.tencent.supersonic.headless.core.translator.calcite.s2sql;
import com.google.common.collect.Lists; import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import java.util.Set;
@Data @Data
public class OntologyQueryParam { public class OntologyQueryParam {
private List<String> metrics = Lists.newArrayList(); private Set<String> metrics = Sets.newHashSet();
private List<String> dimensions = Lists.newArrayList(); private Set<String> dimensions = Sets.newHashSet();
private String where; private String where;
private Long limit; private Long limit;
private List<ColumnOrder> order; private List<ColumnOrder> order;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java
View File

@@ -37,12 +37,6 @@ public class SqlBuilder {
public String buildOntologySql(QueryStatement queryStatement) throws Exception { public String buildOntologySql(QueryStatement queryStatement) throws Exception {
this.ontologyQueryParam = queryStatement.getOntologyQueryParam(); this.ontologyQueryParam = queryStatement.getOntologyQueryParam();
if (ontologyQueryParam.getMetrics() == null) {
ontologyQueryParam.setMetrics(new ArrayList<>());
}
if (ontologyQueryParam.getDimensions() == null) {
ontologyQueryParam.setDimensions(new ArrayList<>());
}
if (ontologyQueryParam.getLimit() == null) { if (ontologyQueryParam.getLimit() == null) {
ontologyQueryParam.setLimit(0L); ontologyQueryParam.setLimit(0L);
} }

45
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java
View File

@@ -150,23 +150,23 @@ public class DataModelNode extends SemanticNode {
} }
public static void getQueryDimensionMeasure(S2CalciteSchema schema, public static void getQueryDimensionMeasure(S2CalciteSchema schema,
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures) { OntologyQueryParam queryParam, Set<String> queryDimensions, Set<String> queryMeasures) {
queryDimension.addAll(metricCommand.getDimensions().stream() queryDimensions.addAll(queryParam.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) .map(d -> d.contains(Constants.DIMENSION_IDENTIFY)
? d.split(Constants.DIMENSION_IDENTIFY)[1] ? d.split(Constants.DIMENSION_IDENTIFY)[1]
: d) : d)
.collect(Collectors.toSet())); .collect(Collectors.toSet()));
Set<String> schemaMetricName = Set<String> schemaMetricName =
schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
schema.getMetrics().stream().filter(m -> metricCommand.getMetrics().contains(m.getName())) schema.getMetrics().stream().filter(m -> queryParam.getMetrics().contains(m.getName()))
.forEach(m -> m.getMetricTypeParams().getMeasures().stream() .forEach(m -> m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> measures.add(mm.getName()))); .forEach(mm -> queryMeasures.add(mm.getName())));
metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)) queryParam.getMetrics().stream().filter(m -> !schemaMetricName.contains(m))
.forEach(m -> measures.add(m)); .forEach(m -> queryMeasures.add(m));
} }
public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema, public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema,
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures, OntologyQueryParam metricCommand, Set<String> queryDimension, Set<String> measures,
SqlValidatorScope scope) throws Exception { SqlValidatorScope scope) throws Exception {
EngineType engineType = schema.getOntology().getDatabase().getType(); EngineType engineType = schema.getOntology().getDatabase().getType();
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
@@ -192,20 +192,20 @@ public class DataModelNode extends SemanticNode {
} }
public static List<DataModel> getRelatedDataModels(SqlValidatorScope scope, public static List<DataModel> getRelatedDataModels(SqlValidatorScope scope,
S2CalciteSchema schema, OntologyQueryParam metricCommand) throws Exception { S2CalciteSchema schema, OntologyQueryParam queryParam) throws Exception {
List<DataModel> dataModels = new ArrayList<>(); List<DataModel> dataModels = new ArrayList<>();
// check by metric // check by metric
List<String> measures = new ArrayList<>(); Set<String> queryMeasures = new HashSet<>();
Set<String> queryDimension = new HashSet<>(); Set<String> queryDimensions = new HashSet<>();
getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures); getQueryDimensionMeasure(schema, queryParam, queryDimensions, queryMeasures);
DataModel baseDataModel = null; DataModel baseDataModel = null;
// one , match measure count // one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>(); Map<String, Integer> dataSourceMeasures = new HashMap<>();
for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) { for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) {
Set<String> sourceMeasure = entry.getValue().getMeasures().stream() Set<String> sourceMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet()); .map(mm -> mm.getName()).collect(Collectors.toSet());
sourceMeasure.retainAll(measures); sourceMeasure.retainAll(queryMeasures);
dataSourceMeasures.put(entry.getKey(), sourceMeasure.size()); dataSourceMeasures.put(entry.getKey(), sourceMeasure.size());
} }
log.info("metrics: [{}]", dataSourceMeasures); log.info("metrics: [{}]", dataSourceMeasures);
@@ -230,17 +230,17 @@ public class DataModelNode extends SemanticNode {
filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(sourceMeasure);
filterMeasure.addAll(dimension); filterMeasure.addAll(dimension);
EngineType engineType = schema.getOntology().getDatabase().getType(); EngineType engineType = schema.getOntology().getDatabase().getType();
mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, mergeQueryFilterDimensionMeasure(schema, queryParam, queryDimensions, queryMeasures,
scope); scope);
boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, boolean isAllMatch = checkMatch(sourceMeasure, queryDimensions, queryMeasures,
metricCommand, scope, engineType); dimension, queryParam, scope, engineType);
if (isAllMatch) { if (isAllMatch) {
log.debug("baseDataModel match all "); log.debug("baseDataModel match all ");
return dataModels; return dataModels;
} }
// find all dataSource has the same identifiers // find all dataSource has the same identifiers
List<DataModel> linkDataModels = getLinkDataSourcesByJoinRelation(queryDimension, List<DataModel> linkDataModels = getLinkDataSourcesByJoinRelation(queryDimensions,
measures, baseDataModel, schema); queryMeasures, baseDataModel, schema);
if (CollectionUtils.isEmpty(linkDataModels)) { if (CollectionUtils.isEmpty(linkDataModels)) {
log.debug("baseDataModel get by identifiers "); log.debug("baseDataModel get by identifiers ");
Set<String> baseIdentifiers = baseDataModel.getIdentifiers().stream() Set<String> baseIdentifiers = baseDataModel.getIdentifiers().stream()
@@ -249,24 +249,23 @@ public class DataModelNode extends SemanticNode {
throw new Exception( throw new Exception(
"datasource error : " + baseDataModel.getName() + " miss identifier"); "datasource error : " + baseDataModel.getName() + " miss identifier");
} }
linkDataModels = getLinkDataSources(baseIdentifiers, queryDimension, measures, linkDataModels = getLinkDataSources(baseIdentifiers, queryDimensions, queryMeasures,
baseDataModel, schema); baseDataModel, schema);
if (linkDataModels.isEmpty()) { if (linkDataModels.isEmpty()) {
throw new Exception(String.format( throw new Exception(String.format(
"not find the match datasource : dimension[%s],measure[%s]", "not find the match datasource : dimension[%s],measure[%s]",
queryDimension, measures)); queryDimensions, queryMeasures));
} }
} }
log.debug("linkDataModels {}", linkDataModels); log.debug("linkDataModels {}", linkDataModels);
return linkDataModels; return linkDataModels;
// dataModels.addAll(linkDataModels);
} }
return dataModels; return dataModels;
} }
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension, private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension,
List<String> measures, Set<String> dimension, OntologyQueryParam metricCommand, Set<String> measures, Set<String> dimension, OntologyQueryParam metricCommand,
SqlValidatorScope scope, EngineType engineType) throws Exception { SqlValidatorScope scope, EngineType engineType) throws Exception {
boolean isAllMatch = true; boolean isAllMatch = true;
sourceMeasure.retainAll(measures); sourceMeasure.retainAll(measures);
@@ -299,7 +298,7 @@ public class DataModelNode extends SemanticNode {
} }
private static List<DataModel> getLinkDataSourcesByJoinRelation(Set<String> queryDimension, private static List<DataModel> getLinkDataSourcesByJoinRelation(Set<String> queryDimension,
List<String> measures, DataModel baseDataModel, S2CalciteSchema schema) { Set<String> measures, DataModel baseDataModel, S2CalciteSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataModel> linkDataModels = new ArrayList<>(); List<DataModel> linkDataModels = new ArrayList<>();
Set<String> before = new HashSet<>(); Set<String> before = new HashSet<>();
@@ -383,7 +382,7 @@ public class DataModelNode extends SemanticNode {
} }
private static List<DataModel> getLinkDataSources(Set<String> baseIdentifiers, private static List<DataModel> getLinkDataSources(Set<String> baseIdentifiers,
Set<String> queryDimension, List<String> measures, DataModel baseDataModel, Set<String> queryDimension, Set<String> measures, DataModel baseDataModel,
S2CalciteSchema schema) { S2CalciteSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataModel> linkDataModels = new ArrayList<>(); List<DataModel> linkDataModels = new ArrayList<>();

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -59,7 +59,7 @@ public class JoinRender extends Renderer {
fieldWhere = whereFields.stream().collect(Collectors.toList()); fieldWhere = whereFields.stream().collect(Collectors.toList());
} }
Set<String> queryAllDimension = new HashSet<>(); Set<String> queryAllDimension = new HashSet<>();
List<String> measures = new ArrayList<>(); Set<String> measures = new HashSet<>();
DataModelNode.getQueryDimensionMeasure(schema, metricCommand, queryAllDimension, measures); DataModelNode.getQueryDimensionMeasure(schema, metricCommand, queryAllDimension, measures);
SqlNode left = null; SqlNode left = null;
TableView leftTable = null; TableView leftTable = null;
@@ -73,8 +73,8 @@ public class JoinRender extends Renderer {
final DataModel dataModel = dataModels.get(i); final DataModel dataModel = dataModels.get(i);
final Set<String> filterDimensions = new HashSet<>(); final Set<String> filterDimensions = new HashSet<>();
final Set<String> filterMetrics = new HashSet<>(); final Set<String> filterMetrics = new HashSet<>();
final List<String> queryDimension = new ArrayList<>(); final Set<String> queryDimension = new HashSet<>();
final List<String> queryMetrics = new ArrayList<>(); final Set<String> queryMetrics = new HashSet<>();
SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataModel, schema, SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataModel, schema,
filterDimensions, filterMetrics); filterDimensions, filterMetrics);
List<String> reqMetric = new ArrayList<>(metricCommand.getMetrics()); List<String> reqMetric = new ArrayList<>(metricCommand.getMetrics());
@@ -142,7 +142,7 @@ public class JoinRender extends Renderer {
} }
private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView, private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView,
List<String> queryMetrics, List<String> reqMetrics, DataModel dataModel, Set<String> queryMetrics, List<String> reqMetrics, DataModel dataModel,
Set<String> sourceMeasure, SqlValidatorScope scope, S2CalciteSchema schema, Set<String> sourceMeasure, SqlValidatorScope scope, S2CalciteSchema schema,
boolean nonAgg) throws Exception { boolean nonAgg) throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName();
@@ -177,7 +177,7 @@ public class JoinRender extends Renderer {
} }
private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension, private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension,
List<String> queryDimension, List<String> reqDimensions, DataModel dataModel, Set<String> queryDimension, List<String> reqDimensions, DataModel dataModel,
Set<String> dimension, SqlValidatorScope scope, S2CalciteSchema schema) Set<String> dimension, SqlValidatorScope scope, S2CalciteSchema schema)
throws Exception { throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName();
@@ -205,7 +205,7 @@ public class JoinRender extends Renderer {
} }
private boolean getMatchMetric(S2CalciteSchema schema, Set<String> sourceMeasure, String m, private boolean getMatchMetric(S2CalciteSchema schema, Set<String> sourceMeasure, String m,
List<String> queryMetrics) { Set<String> queryMetrics) {
Optional<Metric> metric = schema.getMetrics().stream() Optional<Metric> metric = schema.getMetrics().stream()
.filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst(); .filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst();
boolean isAdd = false; boolean isAdd = false;
@@ -226,7 +226,7 @@ public class JoinRender extends Renderer {
} }
private boolean getMatchDimension(S2CalciteSchema schema, Set<String> sourceDimension, private boolean getMatchDimension(S2CalciteSchema schema, Set<String> sourceDimension,
DataModel dataModel, String d, List<String> queryDimension) { DataModel dataModel, String d, Set<String> queryDimension) {
String oriDimension = d; String oriDimension = d;
boolean isAdd = false; boolean isAdd = false;
if (d.contains(Constants.DIMENSION_IDENTIFY)) { if (d.contains(Constants.DIMENSION_IDENTIFY)) {

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -41,14 +41,14 @@ import static com.tencent.supersonic.headless.core.translator.calcite.s2sql.Cons
public class SourceRender extends Renderer { public class SourceRender extends Renderer {
public static TableView renderOne(String alias, List<String> fieldWheres, public static TableView renderOne(String alias, List<String> fieldWheres,
List<String> reqMetrics, List<String> reqDimensions, String queryWhere, Set<String> reqMetrics, Set<String> reqDimensions, String queryWhere,
DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg)
throws Exception { throws Exception {
TableView dataSet = new TableView(); TableView dataSet = new TableView();
TableView output = new TableView(); TableView output = new TableView();
List<String> queryMetrics = new ArrayList<>(reqMetrics); Set<String> queryMetrics = new HashSet<>(reqMetrics);
List<String> queryDimensions = new ArrayList<>(reqDimensions); Set<String> queryDimensions = new HashSet<>(reqDimensions);
List<String> fieldWhere = new ArrayList<>(fieldWheres); List<String> fieldWhere = new ArrayList<>(fieldWheres);
Map<String, String> extendFields = new HashMap<>(); Map<String, String> extendFields = new HashMap<>();
if (!fieldWhere.isEmpty()) { if (!fieldWhere.isEmpty()) {
@@ -57,9 +57,7 @@ public class SourceRender extends Renderer {
whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema, whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema,
dimensions, metrics); dimensions, metrics);
queryMetrics.addAll(metrics); queryMetrics.addAll(metrics);
queryMetrics = uniqList(queryMetrics);
queryDimensions.addAll(dimensions); queryDimensions.addAll(dimensions);
queryDimensions = uniqList(queryDimensions);
mergeWhere(fieldWhere, dataSet, output, queryMetrics, queryDimensions, extendFields, mergeWhere(fieldWhere, dataSet, output, queryMetrics, queryDimensions, extendFields,
datasource, scope, schema, nonAgg); datasource, scope, schema, nonAgg);
} }
@@ -182,8 +180,8 @@ public class SourceRender extends Renderer {
} }
} }
private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics, private static List<SqlNode> getWhereMeasure(List<String> fields, Set<String> queryMetrics,
List<String> queryDimensions, Map<String, String> extendFields, DataModel datasource, Set<String> queryDimensions, Map<String, String> extendFields, DataModel datasource,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
Iterator<String> iterator = fields.iterator(); Iterator<String> iterator = fields.iterator();
List<SqlNode> whereNode = new ArrayList<>(); List<SqlNode> whereNode = new ArrayList<>();
@@ -224,17 +222,17 @@ public class SourceRender extends Renderer {
} }
private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet, private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet,
List<String> queryMetrics, List<String> queryDimensions, Set<String> queryMetrics, Set<String> queryDimensions, Map<String, String> extendFields,
Map<String, String> extendFields, DataModel datasource, SqlValidatorScope scope, DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg)
S2CalciteSchema schema, boolean nonAgg) throws Exception { throws Exception {
List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions,
extendFields, datasource, scope, schema, nonAgg); extendFields, datasource, scope, schema, nonAgg);
dataSet.getMeasure().addAll(whereNode); dataSet.getMeasure().addAll(whereNode);
// getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema); // getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema);
} }
public static void whereDimMetric(List<String> fields, List<String> queryMetrics, public static void whereDimMetric(List<String> fields, Set<String> queryMetrics,
List<String> queryDimensions, DataModel datasource, S2CalciteSchema schema, Set<String> queryDimensions, DataModel datasource, S2CalciteSchema schema,
Set<String> dimensions, Set<String> metrics) { Set<String> dimensions, Set<String> metrics) {
for (String field : fields) { for (String field : fields) {
if (queryDimensions.contains(field) || queryMetrics.contains(field)) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) {
@@ -310,7 +308,7 @@ public class SourceRender extends Renderer {
return false; return false;
} }
private static void addTimeDimension(DataModel dataModel, List<String> queryDimension) { private static void addTimeDimension(DataModel dataModel, Set<String> queryDimension) {
if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) { if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) {
Optional<Dimension> startTimeOp = dataModel.getDimensions().stream() Optional<Dimension> startTimeOp = dataModel.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java
View File

@@ -158,7 +158,6 @@ public class SqlQueryConverter implements QueryConverter {
ontologyParam.getMetrics().addAll(measures); ontologyParam.getMetrics().addAll(measures);
} else { } else {
// empty measure , fill default // empty measure , fill default
ontologyParam.setMetrics(new ArrayList<>());
ontologyParam.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( ontologyParam.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, ontologyParam.getDimensions()))); getDefaultModel(semanticSchemaResp, ontologyParam.getDimensions())));
} }
@@ -168,8 +167,8 @@ public class SqlQueryConverter implements QueryConverter {
} }
private Map<String, String> generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils, private Map<String, String> generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils,
SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List<String> metrics, SemanticSchemaResp semanticSchemaResp, AggOption aggOption, Set<String> metrics,
List<String> dimensions, Set<String> measures) { Set<String> dimensions, Set<String> measures) {
Map<String, String> result = new HashMap<>(); Map<String, String> result = new HashMap<>();
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics(); List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions(); List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
@@ -291,7 +290,7 @@ public class SqlQueryConverter implements QueryConverter {
return elements.stream(); return elements.stream();
} }
private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List<String> dimensions) { private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, Set<String> dimensions) {
if (!CollectionUtils.isEmpty(dimensions)) { if (!CollectionUtils.isEmpty(dimensions)) {
Map<String, Long> modelMatchCnt = new HashMap<>(); Map<String, Long> modelMatchCnt = new HashMap<>();
for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { for (ModelResp modelResp : semanticSchemaResp.getModelResps()) {

42
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -2,33 +2,14 @@ package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.*;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization.TimePartType; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.*;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -36,15 +17,8 @@ import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Slf4j @Slf4j
@@ -184,16 +158,6 @@ public class SemanticSchemaManager {
if (Objects.nonNull(d.getModelSourceTypeEnum())) { if (Objects.nonNull(d.getModelSourceTypeEnum())) {
dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
} }
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> measures = dataModel.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
for (Field f : d.getFields()) {
if (!measures.contains(f.getFieldName())) {
dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName())
.name(f.getFieldName()).agg("").build());
}
}
}
return dataModel; return dataModel;
} }

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java
View File

@@ -9,15 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
@@ -64,5 +56,4 @@ public interface SchemaService {
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
DatabaseResp getDatabase(Long id);
} }

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -536,8 +536,4 @@ public class SchemaServiceImpl implements SchemaService {
return modelService.getItemDate(dimension, metric); return modelService.getItemDate(dimension, metric);
} }
@Override
public DatabaseResp getDatabase(Long id) {
return databaseService.getDatabase(id);
}
} }

9
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashSet;
import java.util.List; import java.util.List;
@Slf4j @Slf4j
@@ -155,8 +156,8 @@ class HeadlessParserServiceTest {
// HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
OntologyQueryParam metricCommand = new OntologyQueryParam(); OntologyQueryParam metricCommand = new OntologyQueryParam();
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv")));
metricCommand.setWhere( metricCommand.setWhere(
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
metricCommand.setLimit(1000L); metricCommand.setLimit(1000L);
@@ -168,9 +169,9 @@ class HeadlessParserServiceTest {
addDepartment(semanticSchema); addDepartment(semanticSchema);
OntologyQueryParam metricCommand2 = new OntologyQueryParam(); OntologyQueryParam metricCommand2 = new OntologyQueryParam();
metricCommand2.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date", metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date",
"user_name__department", "user_name", "user_name__page"))); "user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv")));
metricCommand2.setWhere( metricCommand2.setWhere(
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
metricCommand2.setLimit(1000L); metricCommand2.setLimit(1000L);

4
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
View File

@@ -383,9 +383,9 @@ public class S2VisitsDemo extends S2BaseDemo {
metricReq.setDescription("访问的用户个数"); metricReq.setDescription("访问的用户个数");
metricReq.setAlias("UV,访问人数"); metricReq.setAlias("UV,访问人数");
MetricDefineByFieldParams metricTypeParams = new MetricDefineByFieldParams(); MetricDefineByFieldParams metricTypeParams = new MetricDefineByFieldParams();
metricTypeParams.setExpr("count(distinct user_id)"); metricTypeParams.setExpr("count(distinct user_name)");
List<FieldParam> fieldParams = new ArrayList<>(); List<FieldParam> fieldParams = new ArrayList<>();
fieldParams.add(new FieldParam("user_id")); fieldParams.add(new FieldParam("user_name"));
metricTypeParams.setFields(fieldParams); metricTypeParams.setFields(fieldParams);
metricReq.setMetricDefineByFieldParams(metricTypeParams); metricReq.setMetricDefineByFieldParams(metricTypeParams);
metricReq.setMetricDefineType(MetricDefineType.FIELD); metricReq.setMetricDefineType(MetricDefineType.FIELD);

1
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java
View File

@@ -58,6 +58,7 @@ public class QueryByMetricTest extends BaseTest {
@Test @Test
public void testWithMetricAndDimensionIds() throws Exception { public void testWithMetricAndDimensionIds() throws Exception {
System.setProperty("s2.test", "true");
QueryMetricReq queryMetricReq = new QueryMetricReq(); QueryMetricReq queryMetricReq = new QueryMetricReq();
queryMetricReq.setDomainId(1L); queryMetricReq.setDomainId(1L);
queryMetricReq.setMetricIds(Arrays.asList(1L, 3L)); queryMetricReq.setMetricIds(Arrays.asList(1L, 3L));

2
launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java
View File

@@ -18,7 +18,7 @@ public class TranslateTest extends BaseTest {
public void testSqlExplain() throws Exception { public void testSqlExplain() throws Exception {
String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "; String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 ";
SemanticTranslateResp explain = semanticLayerService.translate( SemanticTranslateResp explain = semanticLayerService.translate(
QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView()), QueryReqBuilder.buildS2SQLReq(sql, DataUtils.productDatasetId),
User.getDefaultUser()); User.getDefaultUser());
assertNotNull(explain); assertNotNull(explain);
assertNotNull(explain.getQuerySQL()); assertNotNull(explain.getQuerySQL());

8
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -18,6 +18,11 @@ public class DataUtils {
public static final Integer productAgentId = 1; public static final Integer productAgentId = 1;
public static final Integer companyAgentId = 2; public static final Integer companyAgentId = 2;
public static final Integer singerAgentId = 3; public static final Integer singerAgentId = 3;
public static final Long productDatasetId = 1L;
public static final Long companyDatasetId = 2L;
public static final Long singerDatasettId = 3L;
public static final Integer ONE_TURNS_CHAT_ID = 10; public static final Integer ONE_TURNS_CHAT_ID = 10;
private static final User user_test = User.getDefaultUser(); private static final User user_test = User.getDefaultUser();
@@ -92,7 +97,4 @@ public class DataUtils {
return result; return result;
} }
public static Long getMetricAgentView() {
return 1L;
}
} }