topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Improve model list performance (#880)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-04-03 18:54:48 +08:00
committed by GitHub
parent 6996c4c12e
commit 12c06c8ebe
16 changed files with 83 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
common/src/main/java/com/tencent/supersonic/common/pojo/enums/PublishEnum.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.common.pojo.enums;
public enum PublishEnum {
UN_PUBLISHED(0),
PUBLISHED(1);
private Integer code;
PublishEnum(Integer code) {
this.code = code;
}
public Integer getCode() {
return code;
}
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java
View File

@@ -7,6 +7,7 @@ import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.List;
import java.util.Map;
@Data
public class DimensionReq extends SchemaItem {
@@ -30,4 +31,6 @@ public class DimensionReq extends SchemaItem {
private DataTypeEnums dataType;
private int isTag;
private Map<String, Object> ext;
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetricBaseReq.java
View File

@@ -7,7 +7,6 @@ import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,7 +28,7 @@ public class MetricBaseReq extends SchemaItem {
private int isTag;
private Map<String, Object> ext = new HashMap<>();
private Map<String, Object> ext;
public String getClassifications() {
if (classifications == null) {

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java
View File

@@ -7,7 +7,9 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import lombok.Data;
import lombok.ToString;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Data
@@ -38,4 +40,6 @@ public class DimensionResp extends SchemaItem {
private int isTag;
private Map<String, Object> ext = new HashMap<>();
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DimensionDO.java
View File

@@ -51,4 +51,6 @@ public class DimensionDO {
private String dataType;
private int isTag;
private String ext;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java
View File

@@ -47,9 +47,15 @@ public class ModelRepositoryImpl implements ModelRepository {
if (!CollectionUtils.isEmpty(modelFilter.getDomainIds())) {
wrapper.lambda().in(ModelDO::getDomainId, modelFilter.getDomainIds());
}
if (modelFilter.getDomainId() != null) {
wrapper.lambda().eq(ModelDO::getDomainId, modelFilter.getDomainId());
}
if (!CollectionUtils.isEmpty(modelFilter.getIds())) {
wrapper.lambda().in(ModelDO::getId, modelFilter.getIds());
}
if (!CollectionUtils.isEmpty(modelFilter.getModelIds())) {
wrapper.lambda().in(ModelDO::getId, modelFilter.getModelIds());
}
if (modelFilter.getIncludesDetail() != null && !modelFilter.getIncludesDetail()) {
wrapper.select(ModelDO.class, modelDO -> !modelDO.getColumn().equals("model_detail"));
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
View File

@@ -34,7 +34,7 @@ public interface ModelService {
List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType);
List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum);
List<ModelResp> getModelAuthList(User user, Long domainId, AuthType authTypeEnum);
List<ModelResp> getModelByDomainIds(List<Long> domainIds);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java
View File

@@ -105,7 +105,7 @@ public class DomainServiceImpl implements DomainService {
List<Long> domainIds = domainWithAuthAll.stream().map(DomainResp::getId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));
}
List<ModelResp> modelResps = modelService.getModelAuthList(user, AuthType.ADMIN);
List<ModelResp> modelResps = modelService.getModelAuthList(user, null, AuthType.ADMIN);
if (!CollectionUtils.isEmpty(modelResps)) {
List<Long> domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));

23
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -294,25 +294,23 @@ public class ModelServiceImpl implements ModelService {
@Override
public List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType) {
List<ModelResp> modelResps = getModelAuthList(user, authType);
List<ModelResp> modelResps = getModelAuthList(user, domainId, authType);
Set<ModelResp> modelRespSet = new HashSet<>(modelResps);
List<ModelResp> modelRespsAuthInheritDomain = getModelRespAuthInheritDomain(user, authType);
List<ModelResp> modelRespsAuthInheritDomain = getModelRespAuthInheritDomain(user, domainId, authType);
modelRespSet.addAll(modelRespsAuthInheritDomain);
if (domainId != null && domainId > 0) {
modelRespSet = modelRespSet.stream().filter(modelResp ->
modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet());
}
return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId))
.collect(Collectors.toList());
}
public List<ModelResp> getModelRespAuthInheritDomain(User user, AuthType authType) {
Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, authType);
if (CollectionUtils.isEmpty(domainResps)) {
public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId, AuthType authType) {
List<Long> domainIds = domainService.getDomainAuthSet(user, authType)
.stream().map(DomainResp::getId)
.collect(Collectors.toList());
if (domainIds.contains(domainId)) {
domainIds = Lists.newArrayList(domainId);
} else {
return Lists.newArrayList();
}
List<Long> domainIds = domainResps.stream().map(DomainResp::getId)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIncludesDetail(false);
modelFilter.setDomainIds(domainIds);
@@ -320,9 +318,10 @@ public class ModelServiceImpl implements ModelService {
}
@Override
public List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum) {
public List<ModelResp> getModelAuthList(User user, Long domainId, AuthType authTypeEnum) {
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIncludesDetail(false);
modelFilter.setDomainId(domainId);
List<ModelResp> modelResps = getModelList(modelFilter);
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
List<ModelResp> modelWithAuth = Lists.newArrayList();

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
View File

@@ -38,6 +38,9 @@ public class DimensionConverter {
if (Objects.nonNull(dimensionReq.getDataType())) {
dimensionDO.setDataType(dimensionReq.getDataType().getType());
}
if (dimensionReq.getExt() != null) {
dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt()));
}
return dimensionDO;
}
@@ -53,6 +56,9 @@ public class DimensionConverter {
if (Objects.nonNull(dimensionReq.getDataType())) {
dimensionDO.setDataType(dimensionReq.getDataType().getType());
}
if (dimensionReq.getExt() != null) {
dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt()));
}
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
return dimensionDO;
}
@@ -76,6 +82,9 @@ public class DimensionConverter {
if (Strings.isNotEmpty(dimensionDO.getDataType())) {
dimensionResp.setDataType(DataTypeEnums.of(dimensionDO.getDataType()));
}
if (dimensionDO.getExt() != null) {
dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
}
dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
return dimensionResp;
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.PublishEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper;
@@ -34,7 +35,10 @@ public class MetricConverter {
metricDO.setClassifications(metricReq.getClassifications());
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
metricDO.setStatus(StatusEnum.ONLINE.getCode());
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
metricDO.setIsPublish(PublishEnum.UN_PUBLISHED.getCode());
if (metricReq.getExt() != null) {
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
}
metricDO.setDefineType(metricReq.getMetricDefineType().name());
return metricDO;
}
@@ -80,7 +84,7 @@ public class MetricConverter {
metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(),
RelateDimension.class));
if (metricDO.getExt() != null) {
metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class));
metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), HashMap.class));
}
metricResp.setTypeEnum(TypeEnums.METRIC);
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {

11
launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java
View File

@@ -26,6 +26,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.DataSetService;
@@ -50,9 +51,10 @@ public class BenchMarkDemoDataLoader {
private ModelService modelService;
@Autowired
private ModelRelaService modelRelaService;
@Autowired
private DataSetService viewService;
@Autowired
private MetricService metricService;
public void doRun() {
try {
@@ -67,6 +69,7 @@ public class BenchMarkDemoDataLoader {
addModelRela_3();
addModelRela_4();
addModelRela_5();
batchPushlishMetric();
} catch (Exception e) {
log.error("Failed to add bench mark demo data", e);
}
@@ -310,4 +313,10 @@ public class BenchMarkDemoDataLoader {
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
private void batchPushlishMetric() {
List<Long> ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
metricService.batchPublish(ids, User.getFakeUser());
}
}

6
launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -121,7 +121,6 @@ public class ModelDemoDataLoader {
addDataSet_2();
addAuthGroup_1();
addAuthGroup_2();
batchPushlishMetric();
} catch (Exception e) {
log.error("Failed to add model demo data", e);
}
@@ -587,11 +586,6 @@ public class ModelDemoDataLoader {
authService.addOrUpdateAuthGroup(authGroupReq);
}
private void batchPushlishMetric() {
List<Long> ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
metricService.batchPublish(ids, User.getFakeUser());
}
private RelateDimension getRelateDimension(List<Long> dimensionIds) {
RelateDimension relateDimension = new RelateDimension();
for (Long id : dimensionIds) {

5
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -290,4 +290,7 @@ COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
--20240325
alter table s2_metric change tags classifications varchar(500) null;
alter table s2_metric add column `is_publish` int(10) DEFAULT NULL COMMENT '是否发布';
update s2_metric set is_publish = 1;
update s2_metric set is_publish = 1;
--20240402
alter table s2_dimension add column `ext` varchar(1000) DEFAULT NULL;

1
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -217,6 +217,7 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` (
`default_values` varchar(500) DEFAULT NULL,
`dim_value_maps` varchar(500) DEFAULT NULL,
`is_tag` INT DEFAULT NULL,
`ext` varchar(1000) DEFAULT NULL,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dimension IS 'dimension information table';

33
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -250,6 +250,7 @@ CREATE TABLE `s2_dimension` (
`default_values` varchar(500) DEFAULT NULL,
`dim_value_maps` varchar(5000) DEFAULT NULL,
`is_tag` int(10) DEFAULT NULL,
`ext` varchar(1000) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='维度表';
@@ -332,7 +333,7 @@ CREATE TABLE `s2_model` (
CREATE TABLE `s2_plugin` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`type` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DASHBOARD,WIDGET,URL',
`view` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`data_set` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`pattern` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`parse_mode` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`parse_mode_config` text COLLATE utf8mb4_unicode_ci,
@@ -343,7 +344,6 @@ CREATE TABLE `s2_plugin` (
`updated_by` varchar(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`config` text CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`comment` text COLLATE utf8mb4_unicode_ci,
ADD COLUMN `data_set` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
@@ -351,7 +351,7 @@ CREATE TABLE `s2_query_stat_info` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询标识',
`model_id` bigint(20) DEFAULT NULL,
`view_id` bigint(20) DEFAULT NULL,
`data_set_id` bigint(20) DEFAULT NULL,
`user` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '执行sql的用户',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`query_type` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询对应的场景',
@@ -482,7 +482,7 @@ CREATE TABLE `s2_app`
`updated_by` varchar(255) null
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE s2_view
CREATE TABLE s2_data_set
(
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id BIGINT,
@@ -491,7 +491,7 @@ CREATE TABLE s2_view
`description` VARCHAR(255),
`status` INT,
alias VARCHAR(255),
view_detail text,
data_set_detail text,
created_at datetime,
created_by VARCHAR(255),
updated_at datetime,
@@ -548,25 +548,4 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
`updated_by` varchar(100) DEFAULT NULL ,
`ext` text DEFAULT NULL ,
PRIMARY KEY (`id`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='查询规则表';
-- 2024-04-02补充mysql脚本
CREATE TABLE IF NOT EXISTS `s2_data_set` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20) ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(255) NOT NULL ,
`status` INT NOT NULL DEFAULT '1' ,
`alias` varchar(255) NOT NULL ,
`data_set_detail` text DEFAULT NULL ,
`created_at` datetime NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` datetime DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`query_config` varchar(3000) DEFAULT NULL ,
`admin` varchar(3000) DEFAULT NULL ,
`admin_org` varchar(3000) DEFAULT NULL ,
`query_type` varchar(100) DEFAULT NULL ,
PRIMARY KEY (`id`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='查询规则表';