From 40bfcdce2c7444ab38021577191a318a5f0312ab Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Tue, 26 Nov 2024 15:21:57 +0800 Subject: [PATCH] [improvement][headless]Clean code logic of headless translator. --- .../calcite/s2sql/OntologyQueryParam.java | 7 +-- .../translator/calcite/sql/SqlBuilder.java | 6 --- .../calcite/sql/node/DataModelNode.java | 45 +++++++++---------- .../calcite/sql/render/JoinRender.java | 14 +++--- .../calcite/sql/render/SourceRender.java | 24 +++++----- .../converter/SqlQueryConverter.java | 7 ++- .../server/manager/SemanticSchemaManager.java | 42 ++--------------- .../server/service/SchemaService.java | 11 +---- .../service/impl/SchemaServiceImpl.java | 4 -- .../calcite/HeadlessParserServiceTest.java | 9 ++-- .../tencent/supersonic/demo/S2VisitsDemo.java | 4 +- .../headless/QueryByMetricTest.java | 1 + .../supersonic/headless/TranslateTest.java | 2 +- .../tencent/supersonic/util/DataUtils.java | 8 ++-- 14 files changed, 65 insertions(+), 119 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java index b399eb486..90711c5da 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java @@ -1,16 +1,17 @@ package com.tencent.supersonic.headless.core.translator.calcite.s2sql; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import lombok.Data; import java.util.List; +import java.util.Set; @Data public class OntologyQueryParam { - private List metrics = Lists.newArrayList(); - private List dimensions = Lists.newArrayList(); + private Set metrics = Sets.newHashSet(); + private Set dimensions = Sets.newHashSet(); private String where; private Long limit; private List order; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java index a3b18076f..570854a07 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java @@ -37,12 +37,6 @@ public class SqlBuilder { public String buildOntologySql(QueryStatement queryStatement) throws Exception { this.ontologyQueryParam = queryStatement.getOntologyQueryParam(); - if (ontologyQueryParam.getMetrics() == null) { - ontologyQueryParam.setMetrics(new ArrayList<>()); - } - if (ontologyQueryParam.getDimensions() == null) { - ontologyQueryParam.setDimensions(new ArrayList<>()); - } if (ontologyQueryParam.getLimit() == null) { ontologyQueryParam.setLimit(0L); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java index 593b73cf5..5d4b38b49 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java @@ -150,23 +150,23 @@ public class DataModelNode extends SemanticNode { } public static void getQueryDimensionMeasure(S2CalciteSchema schema, - OntologyQueryParam metricCommand, Set queryDimension, List measures) { - queryDimension.addAll(metricCommand.getDimensions().stream() + OntologyQueryParam queryParam, Set queryDimensions, Set queryMeasures) { + queryDimensions.addAll(queryParam.getDimensions().stream() .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d) .collect(Collectors.toSet())); Set schemaMetricName = schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet()); - schema.getMetrics().stream().filter(m -> metricCommand.getMetrics().contains(m.getName())) + schema.getMetrics().stream().filter(m -> queryParam.getMetrics().contains(m.getName())) .forEach(m -> m.getMetricTypeParams().getMeasures().stream() - .forEach(mm -> measures.add(mm.getName()))); - metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)) - .forEach(m -> measures.add(m)); + .forEach(mm -> queryMeasures.add(mm.getName()))); + queryParam.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)) + .forEach(m -> queryMeasures.add(m)); } public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema, - OntologyQueryParam metricCommand, Set queryDimension, List measures, + OntologyQueryParam metricCommand, Set queryDimension, Set measures, SqlValidatorScope scope) throws Exception { EngineType engineType = schema.getOntology().getDatabase().getType(); if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { @@ -192,20 +192,20 @@ public class DataModelNode extends SemanticNode { } public static List getRelatedDataModels(SqlValidatorScope scope, - S2CalciteSchema schema, OntologyQueryParam metricCommand) throws Exception { + S2CalciteSchema schema, OntologyQueryParam queryParam) throws Exception { List dataModels = new ArrayList<>(); // check by metric - List measures = new ArrayList<>(); - Set queryDimension = new HashSet<>(); - getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures); + Set queryMeasures = new HashSet<>(); + Set queryDimensions = new HashSet<>(); + getQueryDimensionMeasure(schema, queryParam, queryDimensions, queryMeasures); DataModel baseDataModel = null; // one , match measure count Map dataSourceMeasures = new HashMap<>(); for (Map.Entry entry : schema.getDataModels().entrySet()) { Set sourceMeasure = entry.getValue().getMeasures().stream() .map(mm -> mm.getName()).collect(Collectors.toSet()); - sourceMeasure.retainAll(measures); + sourceMeasure.retainAll(queryMeasures); dataSourceMeasures.put(entry.getKey(), sourceMeasure.size()); } log.info("metrics: [{}]", dataSourceMeasures); @@ -230,17 +230,17 @@ public class DataModelNode extends SemanticNode { filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(dimension); EngineType engineType = schema.getOntology().getDatabase().getType(); - mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, + mergeQueryFilterDimensionMeasure(schema, queryParam, queryDimensions, queryMeasures, scope); - boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, - metricCommand, scope, engineType); + boolean isAllMatch = checkMatch(sourceMeasure, queryDimensions, queryMeasures, + dimension, queryParam, scope, engineType); if (isAllMatch) { log.debug("baseDataModel match all "); return dataModels; } // find all dataSource has the same identifiers - List linkDataModels = getLinkDataSourcesByJoinRelation(queryDimension, - measures, baseDataModel, schema); + List linkDataModels = getLinkDataSourcesByJoinRelation(queryDimensions, + queryMeasures, baseDataModel, schema); if (CollectionUtils.isEmpty(linkDataModels)) { log.debug("baseDataModel get by identifiers "); Set baseIdentifiers = baseDataModel.getIdentifiers().stream() @@ -249,24 +249,23 @@ public class DataModelNode extends SemanticNode { throw new Exception( "datasource error : " + baseDataModel.getName() + " miss identifier"); } - linkDataModels = getLinkDataSources(baseIdentifiers, queryDimension, measures, + linkDataModels = getLinkDataSources(baseIdentifiers, queryDimensions, queryMeasures, baseDataModel, schema); if (linkDataModels.isEmpty()) { throw new Exception(String.format( "not find the match datasource : dimension[%s],measure[%s]", - queryDimension, measures)); + queryDimensions, queryMeasures)); } } log.debug("linkDataModels {}", linkDataModels); return linkDataModels; - // dataModels.addAll(linkDataModels); } return dataModels; } private static boolean checkMatch(Set sourceMeasure, Set queryDimension, - List measures, Set dimension, OntologyQueryParam metricCommand, + Set measures, Set dimension, OntologyQueryParam metricCommand, SqlValidatorScope scope, EngineType engineType) throws Exception { boolean isAllMatch = true; sourceMeasure.retainAll(measures); @@ -299,7 +298,7 @@ public class DataModelNode extends SemanticNode { } private static List getLinkDataSourcesByJoinRelation(Set queryDimension, - List measures, DataModel baseDataModel, S2CalciteSchema schema) { + Set measures, DataModel baseDataModel, S2CalciteSchema schema) { Set linkDataSourceName = new HashSet<>(); List linkDataModels = new ArrayList<>(); Set before = new HashSet<>(); @@ -383,7 +382,7 @@ public class DataModelNode extends SemanticNode { } private static List getLinkDataSources(Set baseIdentifiers, - Set queryDimension, List measures, DataModel baseDataModel, + Set queryDimension, Set measures, DataModel baseDataModel, S2CalciteSchema schema) { Set linkDataSourceName = new HashSet<>(); List linkDataModels = new ArrayList<>(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index c34c6c18b..531a773b4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -59,7 +59,7 @@ public class JoinRender extends Renderer { fieldWhere = whereFields.stream().collect(Collectors.toList()); } Set queryAllDimension = new HashSet<>(); - List measures = new ArrayList<>(); + Set measures = new HashSet<>(); DataModelNode.getQueryDimensionMeasure(schema, metricCommand, queryAllDimension, measures); SqlNode left = null; TableView leftTable = null; @@ -73,8 +73,8 @@ public class JoinRender extends Renderer { final DataModel dataModel = dataModels.get(i); final Set filterDimensions = new HashSet<>(); final Set filterMetrics = new HashSet<>(); - final List queryDimension = new ArrayList<>(); - final List queryMetrics = new ArrayList<>(); + final Set queryDimension = new HashSet<>(); + final Set queryMetrics = new HashSet<>(); SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataModel, schema, filterDimensions, filterMetrics); List reqMetric = new ArrayList<>(metricCommand.getMetrics()); @@ -142,7 +142,7 @@ public class JoinRender extends Renderer { } private void doMetric(Map innerSelect, TableView filterView, - List queryMetrics, List reqMetrics, DataModel dataModel, + Set queryMetrics, List reqMetrics, DataModel dataModel, Set sourceMeasure, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); @@ -177,7 +177,7 @@ public class JoinRender extends Renderer { } private void doDimension(Map innerSelect, Set filterDimension, - List queryDimension, List reqDimensions, DataModel dataModel, + Set queryDimension, List reqDimensions, DataModel dataModel, Set dimension, SqlValidatorScope scope, S2CalciteSchema schema) throws Exception { String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); @@ -205,7 +205,7 @@ public class JoinRender extends Renderer { } private boolean getMatchMetric(S2CalciteSchema schema, Set sourceMeasure, String m, - List queryMetrics) { + Set queryMetrics) { Optional metric = schema.getMetrics().stream() .filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst(); boolean isAdd = false; @@ -226,7 +226,7 @@ public class JoinRender extends Renderer { } private boolean getMatchDimension(S2CalciteSchema schema, Set sourceDimension, - DataModel dataModel, String d, List queryDimension) { + DataModel dataModel, String d, Set queryDimension) { String oriDimension = d; boolean isAdd = false; if (d.contains(Constants.DIMENSION_IDENTIFY)) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java index 3a777a676..399835731 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java @@ -41,14 +41,14 @@ import static com.tencent.supersonic.headless.core.translator.calcite.s2sql.Cons public class SourceRender extends Renderer { public static TableView renderOne(String alias, List fieldWheres, - List reqMetrics, List reqDimensions, String queryWhere, + Set reqMetrics, Set reqDimensions, String queryWhere, DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView dataSet = new TableView(); TableView output = new TableView(); - List queryMetrics = new ArrayList<>(reqMetrics); - List queryDimensions = new ArrayList<>(reqDimensions); + Set queryMetrics = new HashSet<>(reqMetrics); + Set queryDimensions = new HashSet<>(reqDimensions); List fieldWhere = new ArrayList<>(fieldWheres); Map extendFields = new HashMap<>(); if (!fieldWhere.isEmpty()) { @@ -57,9 +57,7 @@ public class SourceRender extends Renderer { whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema, dimensions, metrics); queryMetrics.addAll(metrics); - queryMetrics = uniqList(queryMetrics); queryDimensions.addAll(dimensions); - queryDimensions = uniqList(queryDimensions); mergeWhere(fieldWhere, dataSet, output, queryMetrics, queryDimensions, extendFields, datasource, scope, schema, nonAgg); } @@ -182,8 +180,8 @@ public class SourceRender extends Renderer { } } - private static List getWhereMeasure(List fields, List queryMetrics, - List queryDimensions, Map extendFields, DataModel datasource, + private static List getWhereMeasure(List fields, Set queryMetrics, + Set queryDimensions, Map extendFields, DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); @@ -224,17 +222,17 @@ public class SourceRender extends Renderer { } private static void mergeWhere(List fields, TableView dataSet, TableView outputSet, - List queryMetrics, List queryDimensions, - Map extendFields, DataModel datasource, SqlValidatorScope scope, - S2CalciteSchema schema, boolean nonAgg) throws Exception { + Set queryMetrics, Set queryDimensions, Map extendFields, + DataModel datasource, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) + throws Exception { List whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, scope, schema, nonAgg); dataSet.getMeasure().addAll(whereNode); // getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema); } - public static void whereDimMetric(List fields, List queryMetrics, - List queryDimensions, DataModel datasource, S2CalciteSchema schema, + public static void whereDimMetric(List fields, Set queryMetrics, + Set queryDimensions, DataModel datasource, S2CalciteSchema schema, Set dimensions, Set metrics) { for (String field : fields) { if (queryDimensions.contains(field) || queryMetrics.contains(field)) { @@ -310,7 +308,7 @@ public class SourceRender extends Renderer { return false; } - private static void addTimeDimension(DataModel dataModel, List queryDimension) { + private static void addTimeDimension(DataModel dataModel, Set queryDimension) { if (Materialization.TimePartType.ZIPPER.equals(dataModel.getTimePartType())) { Optional startTimeOp = dataModel.getDimensions().stream() .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java index 9b46c8388..f5d1a7696 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java @@ -158,7 +158,6 @@ public class SqlQueryConverter implements QueryConverter { ontologyParam.getMetrics().addAll(measures); } else { // empty measure , fill default - ontologyParam.setMetrics(new ArrayList<>()); ontologyParam.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( getDefaultModel(semanticSchemaResp, ontologyParam.getDimensions()))); } @@ -168,8 +167,8 @@ public class SqlQueryConverter implements QueryConverter { } private Map generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils, - SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List metrics, - List dimensions, Set measures) { + SemanticSchemaResp semanticSchemaResp, AggOption aggOption, Set metrics, + Set dimensions, Set measures) { Map result = new HashMap<>(); List metricResps = semanticSchemaResp.getMetrics(); List dimensionResps = semanticSchemaResp.getDimensions(); @@ -291,7 +290,7 @@ public class SqlQueryConverter implements QueryConverter { return elements.stream(); } - private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { + private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, Set dimensions) { if (!CollectionUtils.isEmpty(dimensions)) { Map modelMatchCnt = new HashMap<>(); for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 7e5eff936..5a6228b9e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -2,33 +2,14 @@ package com.tencent.supersonic.headless.server.manager; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataType; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.*; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization.TimePartType; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.server.pojo.yaml.*; import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import lombok.extern.slf4j.Slf4j; @@ -36,15 +17,8 @@ import org.apache.commons.lang3.tuple.Triple; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; @Slf4j @@ -184,16 +158,6 @@ public class SemanticSchemaManager { if (Objects.nonNull(d.getModelSourceTypeEnum())) { dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); } - if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) { - Set measures = dataModel.getMeasures().stream().map(mm -> mm.getName()) - .collect(Collectors.toSet()); - for (Field f : d.getFields()) { - if (!measures.contains(f.getFieldName())) { - dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName()) - .name(f.getFieldName()).agg("").build()); - } - } - } return dataModel; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index f7d978bcc..f240bcf66 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -9,15 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; @@ -64,5 +56,4 @@ public interface SchemaService { ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); - DatabaseResp getDatabase(Long id); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index 2c4a62bd5..950db495e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -536,8 +536,4 @@ public class SchemaServiceImpl implements SchemaService { return modelService.getItemDate(dimension, metric); } - @Override - public DatabaseResp getDatabase(Long id) { - return databaseService.getDatabase(id); - } } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index a0a8080da..2c4fe832d 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; @Slf4j @@ -155,8 +156,8 @@ class HeadlessParserServiceTest { // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); OntologyQueryParam metricCommand = new OntologyQueryParam(); - metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); - metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); + metricCommand.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date"))); + metricCommand.setMetrics(new HashSet<>(Arrays.asList("pv"))); metricCommand.setWhere( "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); metricCommand.setLimit(1000L); @@ -168,9 +169,9 @@ class HeadlessParserServiceTest { addDepartment(semanticSchema); OntologyQueryParam metricCommand2 = new OntologyQueryParam(); - metricCommand2.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date", + metricCommand2.setDimensions(new HashSet<>(Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); - metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); + metricCommand2.setMetrics(new HashSet<>(Arrays.asList("pv"))); metricCommand2.setWhere( "user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') "); metricCommand2.setLimit(1000L); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java index 77bc0081b..f80458c31 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java @@ -383,9 +383,9 @@ public class S2VisitsDemo extends S2BaseDemo { metricReq.setDescription("访问的用户个数"); metricReq.setAlias("UV,访问人数"); MetricDefineByFieldParams metricTypeParams = new MetricDefineByFieldParams(); - metricTypeParams.setExpr("count(distinct user_id)"); + metricTypeParams.setExpr("count(distinct user_name)"); List fieldParams = new ArrayList<>(); - fieldParams.add(new FieldParam("user_id")); + fieldParams.add(new FieldParam("user_name")); metricTypeParams.setFields(fieldParams); metricReq.setMetricDefineByFieldParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.FIELD); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java index 7353dd526..c63732666 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java @@ -58,6 +58,7 @@ public class QueryByMetricTest extends BaseTest { @Test public void testWithMetricAndDimensionIds() throws Exception { + System.setProperty("s2.test", "true"); QueryMetricReq queryMetricReq = new QueryMetricReq(); queryMetricReq.setDomainId(1L); queryMetricReq.setMetricIds(Arrays.asList(1L, 3L)); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java index 555711cc5..18d2880bb 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslateTest.java @@ -18,7 +18,7 @@ public class TranslateTest extends BaseTest { public void testSqlExplain() throws Exception { String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 "; SemanticTranslateResp explain = semanticLayerService.translate( - QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView()), + QueryReqBuilder.buildS2SQLReq(sql, DataUtils.productDatasetId), User.getDefaultUser()); assertNotNull(explain); assertNotNull(explain.getQuerySQL()); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index a882293c1..2292d73b5 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -18,6 +18,11 @@ public class DataUtils { public static final Integer productAgentId = 1; public static final Integer companyAgentId = 2; public static final Integer singerAgentId = 3; + + public static final Long productDatasetId = 1L; + public static final Long companyDatasetId = 2L; + public static final Long singerDatasettId = 3L; + public static final Integer ONE_TURNS_CHAT_ID = 10; private static final User user_test = User.getDefaultUser(); @@ -92,7 +97,4 @@ public class DataUtils { return result; } - public static Long getMetricAgentView() { - return 1L; - } }