topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](project) Adding tag abstraction to the dictionary and optimizing related code. (#785)

Browse Source
This commit is contained in:
lexluo09
2024-03-05 18:00:48 +08:00
committed by GitHub
parent ba83c6ca81
commit b8ecfc6a99
27 changed files with 335 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
View File

@@ -107,7 +107,7 @@ public class KnowledgeController {
}
/**
* dailyDictTask-手动离线更新所以字典
* dailyDictTask-手动离线更新所有字典
*/
@PutMapping("/task/all")
public Boolean dailyDictTask(

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java
View File

@@ -19,7 +19,7 @@ public interface TagService {
TagResp getTag(Long id, User user);
List<TagResp> query(TagFilter tagFilter);
List<TagResp> getTags(TagFilter tagFilter);
PageInfo<TagResp> queryPage(TagFilterPage tagFilterPage, User user);

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -50,7 +50,7 @@ import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
import com.tencent.supersonic.headless.server.utils.TagConverter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -72,7 +72,7 @@ public class QueryServiceImpl implements QueryService {
private StatUtils statUtils;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final TagReqConverter tagReqConverter;
private final TagConverter tagConverter;
private final Catalog catalog;
private final AppService appService;
private final QueryCache queryCache;
@@ -84,7 +84,7 @@ public class QueryServiceImpl implements QueryService {
StatUtils statUtils,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
TagReqConverter tagReqConverter, Catalog catalog,
TagConverter tagConverter, Catalog catalog,
AppService appService,
QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager,
@@ -93,7 +93,7 @@ public class QueryServiceImpl implements QueryService {
this.statUtils = statUtils;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.tagReqConverter = tagReqConverter;
this.tagConverter = tagConverter;
this.catalog = catalog;
this.appService = appService;
this.queryCache = queryCache;
@@ -206,7 +206,7 @@ public class QueryServiceImpl implements QueryService {
SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq);
schemaFilterReq.setModelIds(queryTagReq.getModelIds());
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp);
QueryStatement queryStatement = tagConverter.convert(queryTagReq, semanticSchemaResp);
queryStatement.setModelIds(queryTagReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);

49
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.headless.server.service.impl;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.github.pagehelper.PageInfo;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -12,12 +14,14 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -30,10 +34,9 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
@@ -41,16 +44,10 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import com.tencent.supersonic.headless.server.utils.TagConverter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -58,8 +55,11 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Service
@@ -119,21 +119,31 @@ public class SchemaServiceImpl implements SchemaService {
}
public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
List<Long> dataSetIds = filter.getDataSetIds();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(dataSetIds);
metaFilter.setIds(filter.getDataSetIds());
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(metaFilter);
List<Long> modelIds = dataSetResps.stream().map(DataSetResp::getAllModels)
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
List<Long> modelIds = dataSetRespMap.values().stream().map(DataSetResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
metaFilter.setModelIds(modelIds);
metaFilter.setIds(Lists.newArrayList());
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
metaFilter.setIds(modelIds);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(modelIds);
List<TagResp> tagResps = tagService.getTags(tagFilter);
List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
for (Long dataSetId : dataSetRespMap.keySet()) {
DataSetResp dataSetResp = dataSetRespMap.get(dataSetId);
if (dataSetResp == null || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) {
@@ -149,6 +159,9 @@ public class SchemaServiceImpl implements SchemaService {
dataSetSchemaResp.setMetrics(metricSchemaResps);
dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
tagResps = TagConverter.filterByDataSet(tagResps, dataSetResp);
dataSetSchemaResp.setTags(tagResps);
dataSetSchemaResps.add(dataSetSchemaResp);
}
fillStaticInfo(dataSetSchemaResps);
@@ -309,7 +322,7 @@ public class SchemaServiceImpl implements SchemaService {
// add tag info
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(schemaFilterReq.getModelIds());
List<TagResp> tagResps = tagService.query(tagFilter);
List<TagResp> tagResps = tagService.getTags(tagFilter);
semanticSchemaResp.setTags(tagResps);
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java
View File

@@ -123,7 +123,7 @@ public class TagServiceImpl implements TagService {
}
@Override
public List<TagResp> query(TagFilter tagFilter) {
public List<TagResp> getTags(TagFilter tagFilter) {
List<TagDO> tagDOS = tagRepository.query(tagFilter);
if (!CollectionUtils.isEmpty(tagDOS)) {
return tagDOS.stream().map(tagDO -> convert(tagDO)).collect(Collectors.toList());
@@ -154,7 +154,7 @@ public class TagServiceImpl implements TagService {
PageInfo<TagDO> tagDOPageInfo = PageHelper.startPage(tagFilterPage.getCurrent(),
tagFilterPage.getPageSize())
.doSelectPageInfo(() -> query(tagFilter));
.doSelectPageInfo(() -> getTags(tagFilter));
PageInfo<TagResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(tagDOPageInfo, pageInfo);
List<TagResp> tagRespList = convertList(tagDOPageInfo.getList(), collectIds);
@@ -223,7 +223,7 @@ public class TagServiceImpl implements TagService {
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(Arrays.asList(tagReq.getModelId()));
List<TagResp> tagResps = query(tagFilter);
List<TagResp> tagResps = getTags(tagFilter);
if (!CollectionUtils.isEmpty(tagResps)) {
Long bizNameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId()))
.filter(tagResp -> tagResp.getBizName().equalsIgnoreCase(tagReq.getBizName())).count();

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -146,7 +146,7 @@ public class QueryReqConverter {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
if (!SqlSelectHelper.hasGroupBy(sql)
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.NATIVE;

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagConverter.java
View File

@@ -9,11 +9,17 @@ import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -21,13 +27,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Component
@Slf4j
public class TagReqConverter {
public class TagConverter {
@Value("${query.sql.limitWrapper:true}")
private Boolean limitWrapper;
@@ -95,4 +97,9 @@ public class TagReqConverter {
queryParam.setDimensionFilters(queryTagReq.getTagFilters());
queryParam.setQueryType(QueryType.TAG);
}
public static List<TagResp> filterByDataSet(List<TagResp> tagResps, DataSetResp dataSetResp) {
return tagResps.stream().filter(tagResp -> dataSetResp.getAllTags().contains(tagResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(tagResp.getModelId())).collect(Collectors.toList());
}
}