(improvement)(Headless) Improve model list performance (#880)

Co-authored-by: jolunoluo

common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java

@@ -0,0 +1,18 @@
+package com.tencent.supersonic.common.pojo.enums;
+
+public enum PublishEnum {
+
+    UN_PUBLISHED(0),
+    PUBLISHED(1);
+
+    private Integer code;
+
+    PublishEnum(Integer code) {
+        this.code = code;
+    }
+
+    public Integer getCode() {
+        return code;
+    }
+
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java

@@ -7,6 +7,7 @@ import lombok.Data;
 
 import javax.validation.constraints.NotNull;
 import java.util.List;
+import java.util.Map;
 
 @Data
 public class DimensionReq extends SchemaItem {
@@ -30,4 +31,6 @@ public class DimensionReq extends SchemaItem {
     private DataTypeEnums dataType;
 
     private int isTag;
+
+    private Map<String, Object> ext;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java

@@ -7,7 +7,6 @@ import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -29,7 +28,7 @@ public class MetricBaseReq extends SchemaItem {
 
     private int isTag;
 
-    private Map<String, Object> ext = new HashMap<>();
+    private Map<String, Object> ext;
 
     public String getClassifications() {
         if (classifications == null) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -7,7 +7,9 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import lombok.Data;
 import lombok.ToString;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 
 @Data
@@ -38,4 +40,6 @@ public class DimensionResp extends SchemaItem {
 
     private int isTag;
 
+    private Map<String, Object> ext = new HashMap<>();
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DimensionDO.java

@@ -51,4 +51,6 @@ public class DimensionDO {
     private String dataType;
 
     private int isTag;
+
+    private String ext;
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java

@@ -47,9 +47,15 @@ public class ModelRepositoryImpl implements ModelRepository {
         if (!CollectionUtils.isEmpty(modelFilter.getDomainIds())) {
             wrapper.lambda().in(ModelDO::getDomainId, modelFilter.getDomainIds());
         }
+        if (modelFilter.getDomainId() != null) {
+            wrapper.lambda().eq(ModelDO::getDomainId, modelFilter.getDomainId());
+        }
         if (!CollectionUtils.isEmpty(modelFilter.getIds())) {
             wrapper.lambda().in(ModelDO::getId, modelFilter.getIds());
         }
+        if (!CollectionUtils.isEmpty(modelFilter.getModelIds())) {
+            wrapper.lambda().in(ModelDO::getId, modelFilter.getModelIds());
+        }
         if (modelFilter.getIncludesDetail() != null && !modelFilter.getIncludesDetail()) {
             wrapper.select(ModelDO.class, modelDO -> !modelDO.getColumn().equals("model_detail"));
         }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java

@@ -34,7 +34,7 @@ public interface ModelService {
 
     List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType);
 
-    List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum);
+    List<ModelResp> getModelAuthList(User user, Long domainId, AuthType authTypeEnum);
 
     List<ModelResp> getModelByDomainIds(List<Long> domainIds);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java

@@ -105,7 +105,7 @@ public class DomainServiceImpl implements DomainService {
             List<Long> domainIds = domainWithAuthAll.stream().map(DomainResp::getId).collect(Collectors.toList());
             domainWithAuthAll.addAll(getParentDomain(domainIds));
         }
-        List<ModelResp> modelResps = modelService.getModelAuthList(user, AuthType.ADMIN);
+        List<ModelResp> modelResps = modelService.getModelAuthList(user, null, AuthType.ADMIN);
         if (!CollectionUtils.isEmpty(modelResps)) {
             List<Long> domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList());
             domainWithAuthAll.addAll(getParentDomain(domainIds));

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -294,25 +294,23 @@ public class ModelServiceImpl implements ModelService {
 
     @Override
     public List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType) {
-        List<ModelResp> modelResps = getModelAuthList(user, authType);
+        List<ModelResp> modelResps = getModelAuthList(user, domainId, authType);
         Set<ModelResp> modelRespSet = new HashSet<>(modelResps);
-        List<ModelResp> modelRespsAuthInheritDomain = getModelRespAuthInheritDomain(user, authType);
+        List<ModelResp> modelRespsAuthInheritDomain = getModelRespAuthInheritDomain(user, domainId, authType);
         modelRespSet.addAll(modelRespsAuthInheritDomain);
-        if (domainId != null && domainId > 0) {
-            modelRespSet = modelRespSet.stream().filter(modelResp ->
-                    modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet());
-        }
         return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId))
                 .collect(Collectors.toList());
     }
 
-    public List<ModelResp> getModelRespAuthInheritDomain(User user, AuthType authType) {
+    public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId, AuthType authType) {
-        Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, authType);
+        List<Long> domainIds = domainService.getDomainAuthSet(user, authType)
-        if (CollectionUtils.isEmpty(domainResps)) {
+                .stream().map(DomainResp::getId)
+                .collect(Collectors.toList());
+        if (domainIds.contains(domainId)) {
+            domainIds = Lists.newArrayList(domainId);
+        } else {
             return Lists.newArrayList();
         }
-        List<Long> domainIds = domainResps.stream().map(DomainResp::getId)
-                .collect(Collectors.toList());
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setIncludesDetail(false);
         modelFilter.setDomainIds(domainIds);
@@ -320,9 +318,10 @@ public class ModelServiceImpl implements ModelService {
     }
 
     @Override
-    public List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum) {
+    public List<ModelResp> getModelAuthList(User user, Long domainId, AuthType authTypeEnum) {
         ModelFilter modelFilter = new ModelFilter();
         modelFilter.setIncludesDetail(false);
+        modelFilter.setDomainId(domainId);
         List<ModelResp> modelResps = getModelList(modelFilter);
         Set<String> orgIds = userService.getUserAllOrgId(user.getName());
         List<ModelResp> modelWithAuth = Lists.newArrayList();

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -38,6 +38,9 @@ public class DimensionConverter {
         if (Objects.nonNull(dimensionReq.getDataType())) {
             dimensionDO.setDataType(dimensionReq.getDataType().getType());
         }
+        if (dimensionReq.getExt() != null) {
+            dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt()));
+        }
         return dimensionDO;
     }
 
@@ -53,6 +56,9 @@ public class DimensionConverter {
         if (Objects.nonNull(dimensionReq.getDataType())) {
             dimensionDO.setDataType(dimensionReq.getDataType().getType());
         }
+        if (dimensionReq.getExt() != null) {
+            dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt()));
+        }
         dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
         return dimensionDO;
     }
@@ -76,6 +82,9 @@ public class DimensionConverter {
         if (Strings.isNotEmpty(dimensionDO.getDataType())) {
             dimensionResp.setDataType(DataTypeEnums.of(dimensionDO.getDataType()));
         }
+        if (dimensionDO.getExt() != null) {
+            dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
+        }
         dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
         return dimensionResp;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.DataFormat;
+import com.tencent.supersonic.common.pojo.enums.PublishEnum;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
@@ -34,7 +35,10 @@ public class MetricConverter {
         metricDO.setClassifications(metricReq.getClassifications());
         metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
         metricDO.setStatus(StatusEnum.ONLINE.getCode());
-        metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
+        metricDO.setIsPublish(PublishEnum.UN_PUBLISHED.getCode());
+        if (metricReq.getExt() != null) {
+            metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
+        }
         metricDO.setDefineType(metricReq.getMetricDefineType().name());
         return metricDO;
     }
@@ -80,7 +84,7 @@ public class MetricConverter {
         metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(),
                 RelateDimension.class));
         if (metricDO.getExt() != null) {
-            metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class));
+            metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), HashMap.class));
         }
         metricResp.setTypeEnum(TypeEnums.METRIC);
         if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {

launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java

@@ -26,6 +26,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
 import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
 import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
 import com.tencent.supersonic.headless.server.service.DomainService;
+import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelRelaService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.DataSetService;
@@ -50,9 +51,10 @@ public class BenchMarkDemoDataLoader {
     private ModelService modelService;
     @Autowired
     private ModelRelaService modelRelaService;
-
     @Autowired
     private DataSetService viewService;
+    @Autowired
+    private MetricService metricService;
 
     public void doRun() {
         try {
@@ -67,6 +69,7 @@ public class BenchMarkDemoDataLoader {
             addModelRela_3();
             addModelRela_4();
             addModelRela_5();
+            batchPushlishMetric();
         } catch (Exception e) {
             log.error("Failed to add bench mark demo data", e);
         }
@@ -310,4 +313,10 @@ public class BenchMarkDemoDataLoader {
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
+
+    private void batchPushlishMetric() {
+        List<Long> ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
+        metricService.batchPublish(ids, User.getFakeUser());
+    }
+
 }

launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java

@@ -121,7 +121,6 @@ public class ModelDemoDataLoader {
             addDataSet_2();
             addAuthGroup_1();
             addAuthGroup_2();
-            batchPushlishMetric();
         } catch (Exception e) {
             log.error("Failed to add model demo data", e);
         }
@@ -587,11 +586,6 @@ public class ModelDemoDataLoader {
         authService.addOrUpdateAuthGroup(authGroupReq);
     }
 
-    private void batchPushlishMetric() {
-        List<Long> ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
-        metricService.batchPublish(ids, User.getFakeUser());
-    }
-
     private RelateDimension getRelateDimension(List<Long> dimensionIds) {
         RelateDimension relateDimension = new RelateDimension();
         for (Long id : dimensionIds) {

launchers/standalone/src/main/resources/config.update/sql-update.sql

@@ -290,4 +290,7 @@ COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
 --20240325
 alter table s2_metric  change tags classifications varchar(500) null;
 alter table s2_metric  add column `is_publish` int(10) DEFAULT NULL COMMENT '是否发布';
-update s2_metric set is_publish = 1;
+update s2_metric set is_publish = 1;
+
+--20240402
+alter table s2_dimension add column `ext` varchar(1000) DEFAULT NULL;

launchers/standalone/src/main/resources/db/schema-h2.sql

@@ -217,6 +217,7 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` (
     `default_values` varchar(500) DEFAULT NULL,
     `dim_value_maps` varchar(500) DEFAULT NULL,
     `is_tag` INT DEFAULT NULL,
+    `ext` varchar(1000) DEFAULT NULL,
     PRIMARY KEY (`id`)
     );
 COMMENT ON TABLE s2_dimension IS 'dimension information table';

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -250,6 +250,7 @@ CREATE TABLE `s2_dimension` (
                                 `default_values` varchar(500) DEFAULT NULL,
                                 `dim_value_maps` varchar(5000) DEFAULT NULL,
                                 `is_tag` int(10) DEFAULT NULL,
+                                `ext` varchar(1000) DEFAULT NULL,
                                 PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='维度表';
 
@@ -332,7 +333,7 @@ CREATE TABLE `s2_model` (
 CREATE TABLE `s2_plugin` (
                              `id` bigint(20) NOT NULL AUTO_INCREMENT,
                              `type` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DASHBOARD,WIDGET,URL',
-                             `view` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+                             `data_set` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
                              `pattern` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
                              `parse_mode` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
                              `parse_mode_config` text COLLATE utf8mb4_unicode_ci,
@@ -343,7 +344,6 @@ CREATE TABLE `s2_plugin` (
                              `updated_by` varchar(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
                              `config` text CHARACTER SET utf8 COLLATE utf8_unicode_ci,
                              `comment` text COLLATE utf8mb4_unicode_ci,
-                             ADD COLUMN `data_set` varchar(100) DEFAULT NULL,
                              PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
@@ -351,7 +351,7 @@ CREATE TABLE `s2_query_stat_info` (
                                       `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                       `trace_id` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询标识',
                                       `model_id` bigint(20) DEFAULT NULL,
-                                      `view_id` bigint(20) DEFAULT NULL,
+                                      `data_set_id` bigint(20) DEFAULT NULL,
                                       `user` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '执行sql的用户',
                                       `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                                       `query_type` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询对应的场景',
@@ -482,7 +482,7 @@ CREATE TABLE `s2_app`
     `updated_by`     varchar(255) null
 )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
-CREATE TABLE s2_view
+CREATE TABLE s2_data_set
 (
     id          BIGINT AUTO_INCREMENT PRIMARY KEY,
     domain_id   BIGINT,
@@ -491,7 +491,7 @@ CREATE TABLE s2_view
     `description` VARCHAR(255),
     `status`      INT,
     alias       VARCHAR(255),
-    view_detail text,
+    data_set_detail text,
     created_at  datetime,
     created_by  VARCHAR(255),
     updated_at  datetime,
@@ -548,25 +548,4 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
     `updated_by` varchar(100) DEFAULT NULL ,
     `ext` text DEFAULT NULL  ,
     PRIMARY KEY (`id`)
-    ) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='查询规则表';
+    ) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='查询规则表';
-
--- 2024-04-02补充mysql脚本
-CREATE TABLE IF NOT EXISTS `s2_data_set` (
-    `id` bigint(20)   NOT NULL AUTO_INCREMENT,
-    `domain_id` bigint(20) ,
-    `name` varchar(255)  NOT NULL ,
-    `biz_name` varchar(255)  NOT NULL ,
-    `description` varchar(255)  NOT NULL ,
-    `status` INT  NOT NULL DEFAULT '1' ,
-    `alias` varchar(255)  NOT NULL ,
-    `data_set_detail` text DEFAULT NULL  ,
-    `created_at` datetime NOT NULL ,
-    `created_by` varchar(100) NOT NULL ,
-    `updated_at` datetime DEFAULT NULL ,
-    `updated_by` varchar(100) DEFAULT NULL ,
-    `query_config` varchar(3000) DEFAULT NULL ,
-    `admin` varchar(3000) DEFAULT NULL ,
-    `admin_org` varchar(3000) DEFAULT NULL ,
-    `query_type` varchar(100) DEFAULT NULL ,
-    PRIMARY KEY (`id`)
-    ) ENGINE = InnoDB DEFAULT CHARSET = utf8;