From 64615cbef952560c688cc4f964117a988c99fd33 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Mon, 18 Dec 2023 13:18:18 +0800 Subject: [PATCH] (improvement)(semantic) Querying dimension values supports models without time dimensions (#528) Co-authored-by: jolunoluo --- .../supersonic/ModelDemoDataLoader.java | 1 - .../semantic/api/model/pojo/Measure.java | 8 -- .../semantic/api/model/pojo/ModelDetail.java | 13 +++ .../semantic/api/model/request/ModelReq.java | 9 +-- .../api/model/response/ModelResp.java | 11 ++- .../application/DimensionServiceImpl.java | 2 +- .../model/application/ModelServiceImpl.java | 4 +- .../model/domain/utils/ModelConverter.java | 1 - .../query/service/QueryServiceImpl.java | 79 +++++++++---------- 9 files changed, 66 insertions(+), 62 deletions(-) diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index 205548d53..4eeb6d339 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -338,7 +338,6 @@ public class ModelDemoDataLoader { List measures = new ArrayList<>(); Measure measure = new Measure("停留时长", "s2_stay_time_statis_stay_hours", AggOperatorEnum.SUM.getOperator(), 1); - measure.setDatasourceId(3L); measures.add(measure); metricTypeParams.setMeasures(measures); metricReq.setTypeParams(metricTypeParams); diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java index 926373173..66e9ac9f3 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java @@ -26,10 +26,6 @@ public class Measure { private Integer isCreateMetric = 0; - private Long datasourceId; - - private String datasourceName; - public Measure(String name, String bizName, String agg, Integer isCreateMetric) { this.name = name; this.agg = agg; @@ -37,8 +33,4 @@ public class Measure { this.bizName = bizName; } - public Measure(String bizName, Long datasourceId) { - this.bizName = bizName; - this.datasourceId = datasourceId; - } } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java index 1be10ae82..1eabd59f5 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java @@ -1,9 +1,13 @@ package com.tencent.supersonic.semantic.api.model.pojo; +import com.google.common.collect.Lists; +import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; import java.util.List; +import java.util.stream.Collectors; @Data @@ -28,4 +32,13 @@ public class ModelDetail { return sqlQuery; } + public List getTimeDims() { + if (CollectionUtils.isEmpty(dimensions)) { + return Lists.newArrayList(); + } + return dimensions.stream() + .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType())) + .collect(Collectors.toList()); + } + } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java index dc4659cb6..b281b1859 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java @@ -2,17 +2,14 @@ package com.tencent.supersonic.semantic.api.model.request; import com.google.common.collect.Lists; -import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum; import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; -import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @Data @@ -43,12 +40,10 @@ public class ModelReq extends SchemaItem { private List adminOrgs = new ArrayList<>(); public List getTimeDimension() { - if (CollectionUtils.isEmpty(modelDetail.getDimensions())) { + if (modelDetail == null) { return Lists.newArrayList(); } - return modelDetail.getDimensions().stream() - .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType())) - .collect(Collectors.toList()); + return modelDetail.getTimeDims(); } public String getViewer() { diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java index f989e28b0..7165ae6da 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.semantic.api.model.response; import com.google.common.base.Objects; +import com.google.common.collect.Lists; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; +import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.Identify; import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail; -import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; import org.springframework.util.CollectionUtils; @@ -65,6 +67,13 @@ public class ModelResp extends SchemaItem { return null; } + public List getTimeDimension() { + if (modelDetail == null) { + return Lists.newArrayList(); + } + return modelDetail.getTimeDims(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 83d62cbae..cfc13253b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -93,7 +93,7 @@ public class DimensionServiceImpl implements DimensionService { return; } Long modelId = dimensionReqs.get(0).getModelId(); - List dimensionResps = getDimensions(modelId); + List dimensionResps = getDimensionInSameDomain(modelId); Map bizNameMap = dimensionResps.stream() .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1)); Map nameMap = dimensionResps.stream() diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 95dfa96de..77953dda4 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -63,6 +63,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -303,7 +304,8 @@ public class ModelServiceImpl implements ModelService { modelRespSet = modelRespSet.stream().filter(modelResp -> modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet()); } - return new ArrayList<>(modelRespSet); + return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId)) + .collect(Collectors.toList()); } public List getModelRespAuthInheritDomain(User user, AuthType authType) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java index 9d81f079a..a8c0bcd15 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java @@ -105,7 +105,6 @@ public class ModelConverter { } public static MetricReq convert(Measure measure, ModelDO modelDO) { - measure.setDatasourceId(modelDO.getId()); MetricReq metricReq = new MetricReq(); metricReq.setName(measure.getName()); metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", "")); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index c35294033..05717ec2c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -1,19 +1,20 @@ package com.tencent.supersonic.semantic.query.service; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; +import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.pojo.Cache; @@ -25,24 +26,25 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; +import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.query.executor.QueryExecutor; import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation; import com.tencent.supersonic.semantic.query.utils.StatUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + @Service @Slf4j @@ -55,6 +57,7 @@ public class QueryServiceImpl implements QueryService { private final CacheUtils cacheUtils; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; + private final Catalog catalog; @Value("${query.cache.enable:true}") private Boolean cacheEnable; @@ -66,12 +69,14 @@ public class QueryServiceImpl implements QueryService { CacheUtils cacheUtils, QueryUtils queryUtils, QueryReqConverter queryReqConverter, - SemanticQueryEngine semanticQueryEngine) { + SemanticQueryEngine semanticQueryEngine, + Catalog catalog) { this.statUtils = statUtils; this.cacheUtils = cacheUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; this.semanticQueryEngine = semanticQueryEngine; + this.catalog = catalog; } @Override @@ -83,7 +88,7 @@ public class QueryServiceImpl implements QueryService { try { queryStatement = convertToQueryStatement(queryS2SQLReq, user); } catch (Exception e) { - log.info("convertToQueryStatement has a exception:{}", e.toString()); + log.info("convertToQueryStatement has a exception:", e); } log.info("queryStatement:{}", queryStatement); QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement); @@ -200,8 +205,8 @@ public class QueryServiceImpl implements QueryService { @Override @SneakyThrows public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { - QueryStructReq queryStructReq = generateDimValueQueryStruct(queryDimValueReq); - return queryByStruct(queryStructReq, user); + QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq); + return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user); } private void handleGlobalCacheDisable(QueryStructReq queryStructCmd) { @@ -295,37 +300,27 @@ public class QueryServiceImpl implements QueryService { return null; } - private QueryStructReq generateDimValueQueryStruct(QueryDimValueReq queryDimValueReq) { - QueryStructReq queryStructReq = new QueryStructReq(); - - queryStructReq.setModelId(queryDimValueReq.getModelId()); - queryStructReq.setGroups(Collections.singletonList(queryDimValueReq.getDimensionBizName())); - - if (!Objects.isNull(queryDimValueReq.getValue())) { - List dimensionFilters = new ArrayList<>(); - Filter dimensionFilter = new Filter(); - dimensionFilter.setOperator(FilterOperatorEnum.LIKE); - dimensionFilter.setRelation(Filter.Relation.FILTER); - dimensionFilter.setBizName(queryDimValueReq.getDimensionBizName()); - dimensionFilter.setValue(queryDimValueReq.getValue()); - dimensionFilters.add(dimensionFilter); - queryStructReq.setDimensionFilters(dimensionFilters); + private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) { + QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); + List modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); + DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(), + queryDimValueReq.getModelId()); + ModelResp modelResp = modelResps.get(0); + String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName()); + List timeDims = modelResp.getTimeDimension(); + if (CollectionUtils.isNotEmpty(timeDims)) { + sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(), + queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), + queryDimValueReq.getDateInfo().getEndDate()); } - List aggregators = new ArrayList<>(); - queryStructReq.setAggregators(aggregators); - DateConf dateInfo = queryDimValueReq.getDateInfo(); - if (dateInfo == null) { - dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.RECENT); - dateInfo.setUnit(1); - } - queryStructReq.setDateInfo(dateInfo); - queryStructReq.setQueryType(QueryType.TAG); - return queryStructReq; + queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); + queryS2SQLReq.setSql(sql); + return queryS2SQLReq; } private String getKeyByModelIds(List modelIds) { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + return String.join(",", modelIds.stream() + .map(Object::toString).collect(Collectors.toList())); } }