(improvement)(Headless) The model list does not return details to improve performance (#870)

Co-authored-by: jolunoluo

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java

@@ -41,14 +41,13 @@ public class SchemaItem extends RecordInfo {
         }
         SchemaItem that = (SchemaItem) o;
         return Objects.equal(id, that.id) && Objects.equal(name, that.name)
-                && Objects.equal(bizName, that.bizName) && Objects.equal(
+                && Objects.equal(bizName, that.bizName)
-                description, that.description) && Objects.equal(status, that.status)
+                && typeEnum == that.typeEnum;
-                && typeEnum == that.typeEnum && Objects.equal(sensitiveLevel, that.sensitiveLevel);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hashCode(id, name, bizName, description, status, typeEnum, sensitiveLevel);
+        return Objects.hashCode(id, name, bizName, typeEnum);
     }
 
     public static List<String> getAliasList(String alias) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java

@@ -14,6 +14,7 @@ import org.springframework.util.CollectionUtils;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -87,4 +88,19 @@ public class ModelResp extends SchemaItem {
         return fieldSet;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        return super.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), domainId);
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java

@@ -201,21 +201,20 @@ public class S2DataPermissionAspect {
         }
     }
 
-    private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
+    private void doFilterCheckLogic(Set<String> resAuthName, Set<String> sensitiveResReq,
-            Set<String> sensitiveResReq) {
+                                    List<Long> modelIdInDataSet, QueryStructReq queryStructReq) {
         Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
         Set<String> need2Apply = resFilterSet.stream()
                 .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
         Set<String> nameCnSet = new HashSet<>();
-
+        ModelFilter modelFilter = new ModelFilter(false, modelIdInDataSet);
-        Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
+        Map<Long, ModelResp> modelRespMap = modelService.getModelMap(modelFilter);
-        List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
+        List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIdInDataSet));
-        List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
         dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
                 .forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
 
         if (!CollectionUtils.isEmpty(need2Apply)) {
-            List<String> admins = modelService.getModelAdmin(modelIds.get(0));
+            List<String> admins = modelService.getModelAdmin(modelIdInDataSet.get(0));
             log.info("in doFilterLogic, need2Apply:{}", need2Apply);
             String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
             throw new InvalidPermissionException(message);
@@ -247,7 +246,7 @@ public class S2DataPermissionAspect {
         Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
 
         // if sensitive fields without permission are involved in filter, thrown an exception
-        doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
+        doFilterCheckLogic(resAuthSet, sensitiveResReq, modelIdInDataSet, queryStructReq);
 
         // row permission pre-filter
         doRowPermission(queryStructReq, authorizedResource);

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java

@@ -50,6 +50,9 @@ public class ModelRepositoryImpl implements ModelRepository {
         if (!CollectionUtils.isEmpty(modelFilter.getIds())) {
             wrapper.lambda().in(ModelDO::getId, modelFilter.getIds());
         }
+        if (modelFilter.getIncludesDetail() != null && !modelFilter.getIncludesDetail()) {
+            wrapper.select(ModelDO.class, modelDO -> !modelDO.getColumn().equals("model_detail"));
+        }
         return modelDOMapper.selectList(wrapper);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java

@@ -11,4 +11,19 @@ public class ModelFilter extends MetaFilter {
 
     private List<Long> domainIds;
 
+    private Boolean includesDetail = true;
+
+    public ModelFilter() {
+
+    }
+
+    public ModelFilter(Boolean includesDetail) {
+        this.includesDetail = includesDetail;
+    }
+
+    public ModelFilter(Boolean includesDetail, List<Long> ids) {
+        this.includesDetail = includesDetail;
+        this.setIds(ids);
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java

@@ -11,6 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
+import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 
 import java.util.List;
 import java.util.Map;
@@ -23,7 +24,7 @@ public interface ModelService {
 
     List<ModelResp> getModelList(MetaFilter metaFilter);
 
-    Map<Long, ModelResp> getModelMap();
+    Map<Long, ModelResp> getModelMap(ModelFilter modelFilter);
 
     void deleteModel(Long id, User user);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -99,6 +99,7 @@ public class DatabaseServiceImpl implements DatabaseService {
     public void deleteDatabase(Long databaseId) {
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setDatabaseId(databaseId);
+        modelFilter.setIncludesDetail(false);
         List<ModelResp> modelResps = datasourceService.getModelList(modelFilter);
         if (!CollectionUtils.isEmpty(modelResps)) {
             List<String> datasourceNames = modelResps.stream()

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.server.persistence.repository.DimensionRe
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
+import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.service.DataSetService;
 import com.tencent.supersonic.headless.server.service.DatabaseService;
@@ -221,7 +222,7 @@ public class DimensionServiceImpl implements DimensionService {
                 .doSelectPageInfo(() -> queryDimension(dimensionFilter));
         PageInfo<DimensionResp> pageInfo = new PageInfo<>();
         BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo);
-        pageInfo.setList(convertList(dimensionDOPageInfo.getList(), modelService.getModelMap()));
+        pageInfo.setList(convertList(dimensionDOPageInfo.getList()));
         return pageInfo;
     }
 
@@ -232,7 +233,7 @@ public class DimensionServiceImpl implements DimensionService {
     @Override
     public List<DimensionResp> queryDimensions(DimensionsFilter dimensionsFilter) {
         List<DimensionDO> dimensions = dimensionRepository.getDimensions(dimensionsFilter);
-        return convertList(dimensions, modelService.getModelMap());
+        return convertList(dimensions);
     }
 
     @Override
@@ -240,7 +241,7 @@ public class DimensionServiceImpl implements DimensionService {
         DimensionFilter dimensionFilter = new DimensionFilter();
         BeanUtils.copyProperties(metaFilter, dimensionFilter);
         List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
-        List<DimensionResp> dimensionResps = convertList(dimensionDOS, modelService.getModelMap());
+        List<DimensionResp> dimensionResps = convertList(dimensionDOS);
 
         List<Long> dimensionIds = dimensionResps.stream().map(dimensionResp -> dimensionResp.getId())
                 .collect(Collectors.toList());
@@ -298,13 +299,16 @@ public class DimensionServiceImpl implements DimensionService {
         return getDimensions(dimensionFilter);
     }
 
-    private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
+    private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS) {
-            Map<Long, ModelResp> modelRespMap) {
+        List<Long> modelIds = dimensionDOS.stream().map(DimensionDO::getModelId)
+                .collect(Collectors.toList());
+        ModelFilter modelFilter = new ModelFilter(false, modelIds);
+        Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
         List<DimensionResp> dimensionResps = Lists.newArrayList();
         if (!CollectionUtils.isEmpty(dimensionDOS)) {
             dimensionResps = dimensionDOS.stream()
                     .map(dimensionDO -> DimensionConverter
-                            .convert2DimensionResp(dimensionDO, modelRespMap))
+                            .convert2DimensionResp(dimensionDO, modelMap))
                     .collect(Collectors.toList());
         }
         fillTagInfo(dimensionResps);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -46,6 +46,7 @@ import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricFilter;
 import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
 import com.tencent.supersonic.headless.server.pojo.ModelCluster;
+import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.DataSetService;
@@ -401,7 +402,9 @@ public class MetricServiceImpl implements MetricService {
         if (metricDO == null) {
             return null;
         }
-        Map<Long, ModelResp> modelMap = modelService.getModelMap();
+        ModelFilter modelFilter = new ModelFilter(false,
+                Lists.newArrayList(metricDO.getModelId()));
+        Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
         List<CollectDO> collectList = collectService.getCollectList(user.getName());
         List<Long> collect = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
         MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, modelMap, collect);
@@ -526,8 +529,10 @@ public class MetricServiceImpl implements MetricService {
 
     private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
         List<MetricResp> metricResps = Lists.newArrayList();
-
+        List<Long> modelIds = metricDOS.stream().map(MetricDO::getModelId)
-        Map<Long, ModelResp> modelMap = modelService.getModelMap();
+                .collect(Collectors.toList());
+        ModelFilter modelFilter = new ModelFilter(false, modelIds);
+        Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
         if (!CollectionUtils.isEmpty(metricDOS)) {
             metricResps = metricDOS.stream()
                     .map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
@@ -584,7 +589,9 @@ public class MetricServiceImpl implements MetricService {
         //3. choose ModelCluster
         Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
         ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
-
+        if (modelCluster == null) {
+            throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
+        }
         //4. set groups
         List<String> dimensionBizNames = dimensionResps.stream()
                 .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
@@ -604,7 +611,7 @@ public class MetricServiceImpl implements MetricService {
         List<String> metricBizNames = metricResps.stream()
                 .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
                 .map(SchemaItem::getBizName).collect(Collectors.toList());
-        if (org.apache.commons.collections.CollectionUtils.isEmpty(metricBizNames)) {
+        if (CollectionUtils.isEmpty(metricBizNames)) {
             throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
         }
         List<Aggregator> aggregators = new ArrayList<>();

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -154,9 +154,9 @@ public class ModelServiceImpl implements ModelService {
     }
 
     @Override
-    public Map<Long, ModelResp> getModelMap() {
+    public Map<Long, ModelResp> getModelMap(ModelFilter modelFilter) {
         Map<Long, ModelResp> map = new HashMap<>();
-        List<ModelResp> modelResps = getModelList(new ModelFilter());
+        List<ModelResp> modelResps = getModelList(modelFilter);
         if (CollectionUtils.isEmpty(modelResps)) {
             return map;
         }
@@ -311,15 +311,19 @@ public class ModelServiceImpl implements ModelService {
         if (CollectionUtils.isEmpty(domainResps)) {
             return Lists.newArrayList();
         }
-        List<ModelResp> allModelList = getModelList(new ModelFilter());
+        List<Long> domainIds = domainResps.stream().map(DomainResp::getId)
-        Set<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet());
+                .collect(Collectors.toList());
-        return allModelList.stream().filter(modelResp ->
+        ModelFilter modelFilter = new ModelFilter();
-                domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList());
+        modelFilter.setIncludesDetail(false);
+        modelFilter.setDomainIds(domainIds);
+        return getModelList(modelFilter);
     }
 
     @Override
     public List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum) {
-        List<ModelResp> modelResps = getModelList(new ModelFilter());
+        ModelFilter modelFilter = new ModelFilter();
+        modelFilter.setIncludesDetail(false);
+        List<ModelResp> modelResps = getModelList(modelFilter);
         Set<String> orgIds = userService.getUserAllOrgId(user.getName());
         List<ModelResp> modelWithAuth = Lists.newArrayList();
         if (authTypeEnum.equals(AuthType.ADMIN)) {
@@ -342,6 +346,7 @@ public class ModelServiceImpl implements ModelService {
         }
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setDomainIds(domainIds);
+        modelFilter.setIncludesDetail(false);
         List<ModelResp> modelResps = getModelList(modelFilter);
         if (CollectionUtils.isEmpty(modelResps)) {
             return modelResps;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
+import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.service.DataSetService;
 import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -216,7 +217,8 @@ public class SchemaServiceImpl implements SchemaService {
         Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
                 .stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
         List<ModelRela> modelRelas = modelRelaService.getModelRela(modelIds);
-        Map<Long, ModelResp> modelMap = modelService.getModelMap();
+        ModelFilter modelFilter = new ModelFilter(true, modelIds);
+        Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
         for (Long modelId : modelIds) {
             ModelResp modelResp = modelMap.get(modelId);
             if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java

@@ -21,6 +21,7 @@ import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
 import com.tencent.supersonic.headless.server.persistence.repository.TagRepository;
+import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagObjectFilter;
 import com.tencent.supersonic.headless.server.service.CollectService;
@@ -30,6 +31,11 @@ import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.service.TagObjectService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.beans.BeanUtils;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -40,12 +46,6 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Service;
-
 @Service
 @Slf4j
 public class TagMetaServiceImpl implements TagMetaService {
@@ -258,7 +258,10 @@ public class TagMetaServiceImpl implements TagMetaService {
 
     private List<ModelResp> getRelatedModel(TagFilterPageReq tagMarketPageReq) {
         List<ModelResp> modelRespList = new ArrayList<>();
-        Map<Long, ModelResp> modelMap = modelService.getModelMap();
+        ModelFilter modelFilter = new ModelFilter();
+        modelFilter.setDomainIds(tagMarketPageReq.getDomainIds());
+        modelFilter.setIds(tagMarketPageReq.getModelIds());
+        Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
         for (Long modelId : modelMap.keySet()) {
             ModelResp modelResp = modelMap.get(modelId);
             if (Objects.isNull(modelResp)) {
@@ -281,7 +284,10 @@ public class TagMetaServiceImpl implements TagMetaService {
     }
 
     private void fillModelInfo(List<TagResp> tagRespList) {
-        Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap();
+        List<Long> modelIds = tagRespList.stream().map(TagResp::getModelId)
+                .collect(Collectors.toList());
+        ModelFilter modelFilter = new ModelFilter(false, modelIds);
+        Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap(modelFilter);
         tagRespList.stream().forEach(tagResp -> {
             if (Objects.nonNull(modelIdAndRespMap) && modelIdAndRespMap.containsKey(tagResp.getModelId())) {
                 tagResp.setModelName(modelIdAndRespMap.get(tagResp.getModelId()).getName());

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java

@@ -1,15 +1,9 @@
 package com.tencent.supersonic.headless.server.utils;
 
-import java.util.regex.Pattern;
-
 public class NameCheckUtils {
 
     public static boolean containsSpecialCharacters(String str) {
-        if (str == null) {
         return false;
     }
-        String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
-        return Pattern.compile(regex).matcher(str).find();
-    }
 
 }

launchers/standalone/src/test/resources/application-local.yaml

@@ -77,4 +77,4 @@ logging:
 
 inMemoryEmbeddingStore:
   persistent:
-    path: /tmp
+    path: d://

launchers/standalone/src/test/resources/db/schema-h2.sql

@@ -190,6 +190,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
     `relate_dimensions` varchar(500) DEFAULT NULL,
     `ext` LONGVARCHAR DEFAULT NULL  ,
     `define_type` varchar(50)  NOT NULL, -- MEASURE, FIELD, METRIC
+    `is_publish` INT,
     PRIMARY KEY (`id`)
     );
 COMMENT ON TABLE s2_metric IS 'metric information table';
@@ -231,8 +232,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
     PRIMARY KEY (`id`)
 );
 
-create table IF NOT EXISTS s2_canvas
+create table IF NOT EXISTS `s2_canvas` (
-(
     id         INT auto_increment,
     domain_id  INT       null,
     type       varchar(20)  null comment 'model、dimension、metric',
@@ -625,4 +625,3 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
     PRIMARY KEY (`id`)
     );
 COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
-