topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) The model list does not return details to improve performance (#870)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-04-02 11:01:34 +08:00
committed by GitHub
parent 3dcbdb28ad
commit 10501b2d86
15 changed files with 106 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java
View File

@@ -41,14 +41,13 @@ public class SchemaItem extends RecordInfo {
}
SchemaItem that = (SchemaItem) o;
return Objects.equal(id, that.id) && Objects.equal(name, that.name)
&& Objects.equal(bizName, that.bizName) && Objects.equal(
description, that.description) && Objects.equal(status, that.status)
&& typeEnum == that.typeEnum && Objects.equal(sensitiveLevel, that.sensitiveLevel);
&& Objects.equal(bizName, that.bizName)
&& typeEnum == that.typeEnum;
}
@Override
public int hashCode() {
return Objects.hashCode(id, name, bizName, description, status, typeEnum, sensitiveLevel);
return Objects.hashCode(id, name, bizName, typeEnum);
}
public static List<String> getAliasList(String alias) {

16
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java
View File

@@ -14,6 +14,7 @@ import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -87,4 +88,19 @@ public class ModelResp extends SchemaItem {
return fieldSet;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
return super.equals(o);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), domainId);
}
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -201,21 +201,20 @@ public class S2DataPermissionAspect {
}
}
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
private void doFilterCheckLogic(Set<String> resAuthName, Set<String> sensitiveResReq,
List<Long> modelIdInDataSet, QueryStructReq queryStructReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
ModelFilter modelFilter = new ModelFilter(false, modelIdInDataSet);
Map<Long, ModelResp> modelRespMap = modelService.getModelMap(modelFilter);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIdInDataSet));
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
List<String> admins = modelService.getModelAdmin(modelIdInDataSet.get(0));
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
@@ -247,7 +246,7 @@ public class S2DataPermissionAspect {
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(resAuthSet, sensitiveResReq, modelIdInDataSet, queryStructReq);
// row permission pre-filter
doRowPermission(queryStructReq, authorizedResource);

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java
View File

@@ -50,6 +50,9 @@ public class ModelRepositoryImpl implements ModelRepository {
if (!CollectionUtils.isEmpty(modelFilter.getIds())) {
wrapper.lambda().in(ModelDO::getId, modelFilter.getIds());
}
if (modelFilter.getIncludesDetail() != null && !modelFilter.getIncludesDetail()) {
wrapper.select(ModelDO.class, modelDO -> !modelDO.getColumn().equals("model_detail"));
}
return modelDOMapper.selectList(wrapper);
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java
View File

@@ -11,4 +11,19 @@ public class ModelFilter extends MetaFilter {
private List<Long> domainIds;
private Boolean includesDetail = true;
public ModelFilter() {
}
public ModelFilter(Boolean includesDetail) {
this.includesDetail = includesDetail;
}
public ModelFilter(Boolean includesDetail, List<Long> ids) {
this.includesDetail = includesDetail;
this.setIds(ids);
}
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import java.util.List;
import java.util.Map;
@@ -23,7 +24,7 @@ public interface ModelService {
List<ModelResp> getModelList(MetaFilter metaFilter);
Map<Long, ModelResp> getModelMap();
Map<Long, ModelResp> getModelMap(ModelFilter modelFilter);
void deleteModel(Long id, User user);

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -99,6 +99,7 @@ public class DatabaseServiceImpl implements DatabaseService {
public void deleteDatabase(Long databaseId) {
ModelFilter modelFilter = new ModelFilter();
modelFilter.setDatabaseId(databaseId);
modelFilter.setIncludesDetail(false);
List<ModelResp> modelResps = datasourceService.getModelList(modelFilter);
if (!CollectionUtils.isEmpty(modelResps)) {
List<String> datasourceNames = modelResps.stream()

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.server.persistence.repository.DimensionRe
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
@@ -221,7 +222,7 @@ public class DimensionServiceImpl implements DimensionService {
.doSelectPageInfo(() -> queryDimension(dimensionFilter));
PageInfo<DimensionResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo);
pageInfo.setList(convertList(dimensionDOPageInfo.getList(), modelService.getModelMap()));
pageInfo.setList(convertList(dimensionDOPageInfo.getList()));
return pageInfo;
}
@@ -232,7 +233,7 @@ public class DimensionServiceImpl implements DimensionService {
@Override
public List<DimensionResp> queryDimensions(DimensionsFilter dimensionsFilter) {
List<DimensionDO> dimensions = dimensionRepository.getDimensions(dimensionsFilter);
return convertList(dimensions, modelService.getModelMap());
return convertList(dimensions);
}
@Override
@@ -240,7 +241,7 @@ public class DimensionServiceImpl implements DimensionService {
DimensionFilter dimensionFilter = new DimensionFilter();
BeanUtils.copyProperties(metaFilter, dimensionFilter);
List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
List<DimensionResp> dimensionResps = convertList(dimensionDOS, modelService.getModelMap());
List<DimensionResp> dimensionResps = convertList(dimensionDOS);
List<Long> dimensionIds = dimensionResps.stream().map(dimensionResp -> dimensionResp.getId())
.collect(Collectors.toList());
@@ -298,13 +299,16 @@ public class DimensionServiceImpl implements DimensionService {
return getDimensions(dimensionFilter);
}
private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
Map<Long, ModelResp> modelRespMap) {
private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS) {
List<Long> modelIds = dimensionDOS.stream().map(DimensionDO::getModelId)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter(false, modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream()
.map(dimensionDO -> DimensionConverter
.convert2DimensionResp(dimensionDO, modelRespMap))
.convert2DimensionResp(dimensionDO, modelMap))
.collect(Collectors.toList());
}
fillTagInfo(dimensionResps);

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -46,6 +46,7 @@ import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
@@ -401,7 +402,9 @@ public class MetricServiceImpl implements MetricService {
if (metricDO == null) {
return null;
}
Map<Long, ModelResp> modelMap = modelService.getModelMap();
ModelFilter modelFilter = new ModelFilter(false,
Lists.newArrayList(metricDO.getModelId()));
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
List<CollectDO> collectList = collectService.getCollectList(user.getName());
List<Long> collect = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, modelMap, collect);
@@ -526,8 +529,10 @@ public class MetricServiceImpl implements MetricService {
private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap();
List<Long> modelIds = metricDOS.stream().map(MetricDO::getModelId)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter(false, modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps = metricDOS.stream()
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
@@ -584,7 +589,9 @@ public class MetricServiceImpl implements MetricService {
//3. choose ModelCluster
Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
if (modelCluster == null) {
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
}
//4. set groups
List<String> dimensionBizNames = dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
@@ -604,7 +611,7 @@ public class MetricServiceImpl implements MetricService {
List<String> metricBizNames = metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(SchemaItem::getBizName).collect(Collectors.toList());
if (org.apache.commons.collections.CollectionUtils.isEmpty(metricBizNames)) {
if (CollectionUtils.isEmpty(metricBizNames)) {
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
}
List<Aggregator> aggregators = new ArrayList<>();

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -154,9 +154,9 @@ public class ModelServiceImpl implements ModelService {
}
@Override
public Map<Long, ModelResp> getModelMap() {
public Map<Long, ModelResp> getModelMap(ModelFilter modelFilter) {
Map<Long, ModelResp> map = new HashMap<>();
List<ModelResp> modelResps = getModelList(new ModelFilter());
List<ModelResp> modelResps = getModelList(modelFilter);
if (CollectionUtils.isEmpty(modelResps)) {
return map;
}
@@ -311,15 +311,19 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(domainResps)) {
return Lists.newArrayList();
}
List<ModelResp> allModelList = getModelList(new ModelFilter());
Set<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet());
return allModelList.stream().filter(modelResp ->
domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList());
List<Long> domainIds = domainResps.stream().map(DomainResp::getId)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIncludesDetail(false);
modelFilter.setDomainIds(domainIds);
return getModelList(modelFilter);
}
@Override
public List<ModelResp> getModelAuthList(User user, AuthType authTypeEnum) {
List<ModelResp> modelResps = getModelList(new ModelFilter());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIncludesDetail(false);
List<ModelResp> modelResps = getModelList(modelFilter);
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
List<ModelResp> modelWithAuth = Lists.newArrayList();
if (authTypeEnum.equals(AuthType.ADMIN)) {
@@ -342,6 +346,7 @@ public class ModelServiceImpl implements ModelService {
}
ModelFilter modelFilter = new ModelFilter();
modelFilter.setDomainIds(domainIds);
modelFilter.setIncludesDetail(false);
List<ModelResp> modelResps = getModelList(modelFilter);
if (CollectionUtils.isEmpty(modelResps)) {
return modelResps;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -34,6 +34,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -216,7 +217,8 @@ public class SchemaServiceImpl implements SchemaService {
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
List<ModelRela> modelRelas = modelRelaService.getModelRela(modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap();
ModelFilter modelFilter = new ModelFilter(true, modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
for (Long modelId : modelIds) {
ModelResp modelResp = modelMap.get(modelId);
if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) {

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java
View File

@@ -21,6 +21,7 @@ import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.persistence.repository.TagRepository;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.pojo.TagObjectFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
@@ -30,6 +31,11 @@ import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TagObjectService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Arrays;
@@ -40,12 +46,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class TagMetaServiceImpl implements TagMetaService {
@@ -258,7 +258,10 @@ public class TagMetaServiceImpl implements TagMetaService {
private List<ModelResp> getRelatedModel(TagFilterPageReq tagMarketPageReq) {
List<ModelResp> modelRespList = new ArrayList<>();
Map<Long, ModelResp> modelMap = modelService.getModelMap();
ModelFilter modelFilter = new ModelFilter();
modelFilter.setDomainIds(tagMarketPageReq.getDomainIds());
modelFilter.setIds(tagMarketPageReq.getModelIds());
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
for (Long modelId : modelMap.keySet()) {
ModelResp modelResp = modelMap.get(modelId);
if (Objects.isNull(modelResp)) {
@@ -281,7 +284,10 @@ public class TagMetaServiceImpl implements TagMetaService {
}
private void fillModelInfo(List<TagResp> tagRespList) {
Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap();
List<Long> modelIds = tagRespList.stream().map(TagResp::getModelId)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter(false, modelIds);
Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap(modelFilter);
tagRespList.stream().forEach(tagResp -> {
if (Objects.nonNull(modelIdAndRespMap) && modelIdAndRespMap.containsKey(tagResp.getModelId())) {
tagResp.setModelName(modelIdAndRespMap.get(tagResp.getModelId()).getName());

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/NameCheckUtils.java
View File

@@ -1,15 +1,9 @@
package com.tencent.supersonic.headless.server.utils;
import java.util.regex.Pattern;
public class NameCheckUtils {
public static boolean containsSpecialCharacters(String str) {
if (str == null) {
return false;
}
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
return Pattern.compile(regex).matcher(str).find();
return false;
}
}

2
launchers/standalone/src/test/resources/application-local.yaml
View File

@@ -77,4 +77,4 @@ logging:
inMemoryEmbeddingStore:
persistent:
path: /tmp
path: d://

13
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -170,9 +170,9 @@ create table IF NOT EXISTS s2_auth_groups
);
CREATE TABLE IF NOT EXISTS `s2_metric` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
@@ -190,6 +190,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`relate_dimensions` varchar(500) DEFAULT NULL,
`ext` LONGVARCHAR DEFAULT NULL ,
`define_type` varchar(50) NOT NULL, -- MEASURE, FIELD, METRIC
`is_publish` INT,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_metric IS 'metric information table';
@@ -231,8 +232,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
PRIMARY KEY (`id`)
);
create table IF NOT EXISTS s2_canvas
(
create table IF NOT EXISTS `s2_canvas` (
id INT auto_increment,
domain_id INT null,
type varchar(20) null comment 'model、dimension、metric',
@@ -624,5 +624,4 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
COMMENT ON TABLE s2_query_rule IS 'tag query rule table';