topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 04:57:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add TagTest and correctDateConf logic (#799)

Browse Source
This commit is contained in:
daikon
2024-03-09 09:19:51 +08:00
committed by GitHub
parent eeb54e0f73
commit f6ff0d19da
7 changed files with 88 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java
View File

@@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import lombok.Data;
@Data
@@ -9,5 +11,6 @@ public class TagFilter extends MetaFilter {
private String type;
private List<Integer> statusList;
private TagDefineType tagDefineType;
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilterPage.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.PageSchemaItemReq;
import java.util.List;
@@ -8,4 +9,5 @@ import java.util.List;
public class TagFilterPage extends PageSchemaItemReq {
private String type;
private List<Integer> statusList;
private TagDefineType tagDefineType;
}

44
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -18,6 +18,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
@@ -26,16 +27,19 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -50,6 +54,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Service
@@ -69,22 +74,26 @@ public class DimensionServiceImpl implements DimensionService {
private DataSetService dataSetService;
private TagMetaService tagMetaService;
@Autowired
private ApplicationEventPublisher eventPublisher;
public DimensionServiceImpl(DimensionRepository dimensionRepository,
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
DataSetService dataSetService) {
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
DataSetService dataSetService,
TagMetaService tagMetaService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.dataSetService = dataSetService;
this.tagMetaService = tagMetaService;
}
@Override
@@ -115,7 +124,7 @@ public class DimensionServiceImpl implements DimensionService {
return;
}
List<DimensionDO> dimensionDOS = dimensionToInsert.stream().peek(dimension ->
dimension.createdBy(user.getName()))
dimension.createdBy(user.getName()))
.map(DimensionConverter::convert2DimensionDO)
.collect(Collectors.toList());
dimensionRepository.createDimensionBatch(dimensionDOS);
@@ -206,7 +215,7 @@ public class DimensionServiceImpl implements DimensionService {
BeanUtils.copyProperties(pageDimensionReq, dimensionFilter);
dimensionFilter.setModelIds(pageDimensionReq.getModelIds());
PageInfo<DimensionDO> dimensionDOPageInfo = PageHelper.startPage(pageDimensionReq.getCurrent(),
pageDimensionReq.getPageSize())
pageDimensionReq.getPageSize())
.doSelectPageInfo(() -> queryDimension(dimensionFilter));
PageInfo<DimensionResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo);
@@ -272,7 +281,7 @@ public class DimensionServiceImpl implements DimensionService {
}
private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
Map<Long, ModelResp> modelRespMap) {
Map<Long, ModelResp> modelRespMap) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream()
@@ -280,9 +289,28 @@ public class DimensionServiceImpl implements DimensionService {
.convert2DimensionResp(dimensionDO, modelRespMap))
.collect(Collectors.toList());
}
fillTagInfo(dimensionResps);
return dimensionResps;
}
private void fillTagInfo(List<DimensionResp> dimensionResps) {
if (CollectionUtils.isEmpty(dimensionResps)) {
return;
}
TagFilter tagFilter = new TagFilter();
tagFilter.setTagDefineType(TagDefineType.DIMENSION);
Map<String, TagResp> keyAndTagMap = tagMetaService.getTags(tagFilter).stream()
.collect(Collectors.toMap(tag -> tag.getModelId() + "_" + tag.getBizName(), tag -> tag));
if (Objects.nonNull(keyAndTagMap)) {
dimensionResps.stream().forEach(dim -> {
String key = dim.getModelId() + "_" + dim.getBizName();
if (keyAndTagMap.containsKey(key)) {
dim.setIsTag(1);
}
});
}
}
@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(),

51
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
@@ -33,6 +34,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
@@ -42,11 +44,13 @@ import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
@@ -64,6 +68,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -85,13 +90,16 @@ public class MetricServiceImpl implements MetricService {
private ApplicationEventPublisher eventPublisher;
private TagMetaService tagMetaService;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService) {
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService,
TagMetaService tagMetaService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.chatGptHelper = chatGptHelper;
@@ -99,6 +107,7 @@ public class MetricServiceImpl implements MetricService {
this.collectService = collectService;
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
this.tagMetaService = tagMetaService;
}
@Override
@@ -211,12 +220,13 @@ public class MetricServiceImpl implements MetricService {
}
}
PageInfo<MetricDO> metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(),
pageMetricReq.getPageSize())
pageMetricReq.getPageSize())
.doSelectPageInfo(() -> queryMetric(metricFilter));
PageInfo<MetricResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(metricDOPageInfo, pageInfo);
List<MetricResp> metricResps = convertList(metricDOPageInfo.getList(), collectIds);
fillAdminRes(metricResps, user);
fillTagInfo(metricResps);
pageInfo.setList(metricResps);
return pageInfo;
}
@@ -240,6 +250,24 @@ public class MetricServiceImpl implements MetricService {
return metricResps;
}
private void fillTagInfo(List<MetricResp> metricRespList) {
if (CollectionUtils.isEmpty(metricRespList)) {
return;
}
TagFilter tagFilter = new TagFilter();
tagFilter.setTagDefineType(TagDefineType.METRIC);
Map<String, TagResp> keyAndTagMap = tagMetaService.getTags(tagFilter).stream()
.collect(Collectors.toMap(tag -> tag.getModelId() + "_" + tag.getBizName(), tag -> tag));
if (Objects.nonNull(keyAndTagMap)) {
metricRespList.stream().forEach(metric -> {
String key = metric.getModelId() + "_" + metric.getBizName();
if (keyAndTagMap.containsKey(key)) {
metric.setIsTag(1);
}
});
}
}
private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
Set<MetricResp> metricRespFiltered = Sets.newHashSet();
for (MetricResp metricResp : metricResps) {
@@ -249,7 +277,7 @@ public class MetricServiceImpl implements MetricService {
}
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) {
List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
@@ -287,8 +315,8 @@ public class MetricServiceImpl implements MetricService {
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream().filter(metricResp ->
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList());
}
@@ -450,6 +478,7 @@ public class MetricServiceImpl implements MetricService {
private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap();
if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps = metricDOS.stream()
@@ -560,7 +589,7 @@ public class MetricServiceImpl implements MetricService {
}
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) {
List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
result.addAll(modelIdsByDomainId);

3
headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml
View File

@@ -67,6 +67,9 @@
<if test="type != null and type != ''">
and type = #{type}
</if>
<if test="tagDefineType != null">
and define_type = #{tagDefineType}
</if>
<if test="key != null and key != ''">
and ( id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or

3
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -69,8 +69,9 @@ public class MetricServiceImplTest {
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, dataSetService,
eventPublisher, dimensionService);
eventPublisher, dimensionService, tagMetaService);
}
private MetricReq buildMetricReq() {