(improvement)(semantic) Querying dimension values supports models without time dimensions (#528)

Co-authored-by: jolunoluo

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Measure.java

@@ -26,10 +26,6 @@ public class Measure {
 
     private Integer isCreateMetric = 0;
 
-    private Long datasourceId;
-
-    private String datasourceName;
-
     public Measure(String name, String bizName, String agg, Integer isCreateMetric) {
         this.name = name;
         this.agg = agg;
@@ -37,8 +33,4 @@ public class Measure {
         this.bizName = bizName;
     }
 
-    public Measure(String bizName, Long datasourceId) {
-        this.bizName = bizName;
-        this.datasourceId = datasourceId;
-    }
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/ModelDetail.java

@@ -1,9 +1,13 @@
 package com.tencent.supersonic.semantic.api.model.pojo;
 
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.CollectionUtils;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 
 @Data
@@ -28,4 +32,13 @@ public class ModelDetail {
         return sqlQuery;
     }
 
+    public List<Dim> getTimeDims() {
+        if (CollectionUtils.isEmpty(dimensions)) {
+            return Lists.newArrayList();
+        }
+        return dimensions.stream()
+                .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType()))
+                .collect(Collectors.toList());
+    }
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java

@@ -2,17 +2,14 @@ package com.tencent.supersonic.semantic.api.model.request;
 
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
 import com.tencent.supersonic.semantic.api.model.pojo.Dim;
 import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
 import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import lombok.Data;
-import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 
 @Data
@@ -43,12 +40,10 @@ public class ModelReq extends SchemaItem {
     private List<String> adminOrgs = new ArrayList<>();
 
     public List<Dim> getTimeDimension() {
-        if (CollectionUtils.isEmpty(modelDetail.getDimensions())) {
+        if (modelDetail == null) {
             return Lists.newArrayList();
         }
-        return modelDetail.getDimensions().stream()
-                .filter(dim -> DimensionTypeEnum.time.name().equalsIgnoreCase(dim.getType()))
-                .collect(Collectors.toList());
+        return modelDetail.getTimeDims();
     }
 
     public String getViewer() {

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ModelResp.java

@@ -1,9 +1,11 @@
 package com.tencent.supersonic.semantic.api.model.response;
 
 import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.semantic.api.model.pojo.Dim;
+import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
 import com.tencent.supersonic.semantic.api.model.pojo.Identify;
 import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
-import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import lombok.Data;
 import org.springframework.util.CollectionUtils;
@@ -65,6 +67,13 @@ public class ModelResp extends SchemaItem {
         return null;
     }
 
+    public List<Dim> getTimeDimension() {
+        if (modelDetail == null) {
+            return Lists.newArrayList();
+        }
+        return modelDetail.getTimeDims();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java

@@ -93,7 +93,7 @@ public class DimensionServiceImpl implements DimensionService {
             return;
         }
         Long modelId = dimensionReqs.get(0).getModelId();
-        List<DimensionResp> dimensionResps = getDimensions(modelId);
+        List<DimensionResp> dimensionResps = getDimensionInSameDomain(modelId);
         Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
                 .collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
         Map<String, DimensionResp> nameMap = dimensionResps.stream()

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java

@@ -63,6 +63,7 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -303,7 +304,8 @@ public class ModelServiceImpl implements ModelService {
             modelRespSet = modelRespSet.stream().filter(modelResp ->
                     modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet());
         }
-        return new ArrayList<>(modelRespSet);
+        return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId))
+                .collect(Collectors.toList());
     }
 
     public List<ModelResp> getModelRespAuthInheritDomain(User user, AuthType authType) {

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConverter.java

@@ -105,7 +105,6 @@ public class ModelConverter {
     }
 
     public static MetricReq convert(Measure measure, ModelDO modelDO) {
-        measure.setDatasourceId(modelDO.getId());
         MetricReq metricReq = new MetricReq();
         metricReq.setName(measure.getName());
         metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", ""));

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java

@@ -1,19 +1,20 @@
 package com.tencent.supersonic.semantic.query.service;
 
 import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.cache.CacheUtils;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
+import com.tencent.supersonic.semantic.api.model.pojo.Dim;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
+import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
+import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.query.pojo.Cache;
@@ -25,24 +26,25 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
+import com.tencent.supersonic.semantic.model.domain.Catalog;
 import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
 import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
 import com.tencent.supersonic.semantic.query.utils.QueryUtils;
 import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation;
 import com.tencent.supersonic.semantic.query.utils.StatUtils;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
 
 @Service
 @Slf4j
@@ -55,6 +57,7 @@ public class QueryServiceImpl implements QueryService {
     private final CacheUtils cacheUtils;
     private final QueryUtils queryUtils;
     private final QueryReqConverter queryReqConverter;
+    private final Catalog catalog;
 
     @Value("${query.cache.enable:true}")
     private Boolean cacheEnable;
@@ -66,12 +69,14 @@ public class QueryServiceImpl implements QueryService {
             CacheUtils cacheUtils,
             QueryUtils queryUtils,
             QueryReqConverter queryReqConverter,
-            SemanticQueryEngine semanticQueryEngine) {
+            SemanticQueryEngine semanticQueryEngine,
+            Catalog catalog) {
         this.statUtils = statUtils;
         this.cacheUtils = cacheUtils;
         this.queryUtils = queryUtils;
         this.queryReqConverter = queryReqConverter;
         this.semanticQueryEngine = semanticQueryEngine;
+        this.catalog = catalog;
     }
 
     @Override
@@ -83,7 +88,7 @@ public class QueryServiceImpl implements QueryService {
         try {
             queryStatement = convertToQueryStatement(queryS2SQLReq, user);
         } catch (Exception e) {
-            log.info("convertToQueryStatement has a exception:{}", e.toString());
+            log.info("convertToQueryStatement has a exception:", e);
         }
         log.info("queryStatement:{}", queryStatement);
         QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement);
@@ -200,8 +205,8 @@ public class QueryServiceImpl implements QueryService {
     @Override
     @SneakyThrows
     public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
-        QueryStructReq queryStructReq = generateDimValueQueryStruct(queryDimValueReq);
-        return queryByStruct(queryStructReq, user);
+        QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq);
+        return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user);
     }
 
     private void handleGlobalCacheDisable(QueryStructReq queryStructCmd) {
@@ -295,37 +300,27 @@ public class QueryServiceImpl implements QueryService {
         return null;
     }
 
-    private QueryStructReq generateDimValueQueryStruct(QueryDimValueReq queryDimValueReq) {
-        QueryStructReq queryStructReq = new QueryStructReq();
-
-        queryStructReq.setModelId(queryDimValueReq.getModelId());
-        queryStructReq.setGroups(Collections.singletonList(queryDimValueReq.getDimensionBizName()));
-
-        if (!Objects.isNull(queryDimValueReq.getValue())) {
-            List<Filter> dimensionFilters = new ArrayList<>();
-            Filter dimensionFilter = new Filter();
-            dimensionFilter.setOperator(FilterOperatorEnum.LIKE);
-            dimensionFilter.setRelation(Filter.Relation.FILTER);
-            dimensionFilter.setBizName(queryDimValueReq.getDimensionBizName());
-            dimensionFilter.setValue(queryDimValueReq.getValue());
-            dimensionFilters.add(dimensionFilter);
-            queryStructReq.setDimensionFilters(dimensionFilters);
+    private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) {
+        QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
+        List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
+        DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
+                queryDimValueReq.getModelId());
+        ModelResp modelResp = modelResps.get(0);
+        String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName());
+        List<Dim> timeDims = modelResp.getTimeDimension();
+        if (CollectionUtils.isNotEmpty(timeDims)) {
+            sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
+                    queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
+                    queryDimValueReq.getDateInfo().getEndDate());
         }
-        List<Aggregator> aggregators = new ArrayList<>();
-        queryStructReq.setAggregators(aggregators);
-        DateConf dateInfo = queryDimValueReq.getDateInfo();
-        if (dateInfo == null) {
-            dateInfo = new DateConf();
-            dateInfo.setDateMode(DateConf.DateMode.RECENT);
-            dateInfo.setUnit(1);
-        }
-        queryStructReq.setDateInfo(dateInfo);
-        queryStructReq.setQueryType(QueryType.TAG);
-        return queryStructReq;
+        queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
+        queryS2SQLReq.setSql(sql);
+        return queryS2SQLReq;
     }
 
     private String getKeyByModelIds(List<Long> modelIds) {
-        return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
+        return String.join(",", modelIds.stream()
+                .map(Object::toString).collect(Collectors.toList()));
     }
 
 }