From 26bef3d472723e809f459cbfdfa2130fd03aba1e Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Tue, 12 Mar 2024 21:28:02 +0800 Subject: [PATCH] add constructTagQueryReq for dict (#807) --- .../api/pojo/request/SemanticQueryReq.java | 3 + .../api/pojo/response/DictItemResp.java | 3 +- .../service/impl/TagMetaServiceImpl.java | 11 +-- .../headless/server/utils/DictUtils.java | 72 +++++++++++++++++-- .../tencent/supersonic/headless/TagTest.java | 33 +++++++++ 5 files changed, 110 insertions(+), 12 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index 19976b134..3f2ab6e21 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Param; import lombok.Data; @@ -23,6 +24,8 @@ public abstract class SemanticQueryReq { protected String dataSetName; + protected QueryType queryType; + protected Set modelIds = new HashSet<>(); protected List params = new ArrayList<>(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java index fe1cc7bde..65e531b2c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java @@ -32,7 +32,7 @@ public class DictItemResp { @NotNull private StatusEnum status; - public String getNature() { + public String generateNature() { return UNDERLINE + modelId + UNDERLINE + itemId + UNDERLINE + type.name().toLowerCase().substring(0, 1) + DICT_VALUE; @@ -41,4 +41,5 @@ public class DictItemResp { public String fetchDictFileName() { return String.format("dic_value_%d_%s_%s", modelId, type.name(), itemId); } + } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java index b6788d56f..96e39764f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java @@ -231,14 +231,16 @@ public class TagMetaServiceImpl implements TagMetaService { public Integer createBatch(TagBatchCreateReq tagLoadReq, User user) { Long modelId = tagLoadReq.getModelId(); int num = 0; + MetaFilter metaFilter = new MetaFilter(); + List modelIds = new ArrayList<>(); + modelIds.add(modelId); + metaFilter.setModelIds(modelIds); if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.DIMENSION.equals(tagLoadReq.getType())) { - List dimensions = dimensionService.getDimensionInModelCluster(modelId); + List dimensions = dimensionService.getDimensions(metaFilter); num += loadDimTagBatch(tagLoadReq, dimensions, user); } if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.METRIC.equals(tagLoadReq.getType())) { - MetaFilter metaFilter = new MetaFilter(); - List modelIds = new ArrayList<>(); - modelIds.add(modelId); + List metrics = metricService.getMetrics(metaFilter); num += loadMetricTagBatch(tagLoadReq, metrics, user); } @@ -369,6 +371,7 @@ public class TagMetaServiceImpl implements TagMetaService { if (CollectionUtils.isNotEmpty(tagDOList)) { tagDOList.stream().forEach(tagDO -> { TagResp tagResp = convert(tagDO); + tagResp.setTypeEnum(TypeEnums.TAG); if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagDO.getId())) { tagResp.setIsCollect(true); } else { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 9c5ada64d..4d1f6a0a1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -15,6 +15,7 @@ import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; @@ -25,12 +26,14 @@ import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -51,11 +54,15 @@ import java.util.Objects; import java.util.Set; import java.util.StringJoiner; +import com.tencent.supersonic.headless.server.service.TagMetaService; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +@Slf4j @Component public class DictUtils { @@ -79,15 +86,18 @@ public class DictUtils { private final MetricService metricService; private final QueryService queryService; private final ModelService modelService; + private final TagMetaService tagMetaService; public DictUtils(DimensionService dimensionService, MetricService metricService, QueryService queryService, - ModelService modelService) { + ModelService modelService, + @Lazy TagMetaService tagMetaService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; this.modelService = modelService; + this.tagMetaService = tagMetaService; } public String fetchDictFileName(DictItemResp dictItemResp) { @@ -140,17 +150,22 @@ public class DictUtils { dictItemResp.setModelId(dimension.getModelId()); dictItemResp.setBizName(dimension.getBizName()); } + if (TypeEnums.TAG.equals(TypeEnums.valueOf(dictConfDO.getType()))) { + TagResp tagResp = tagMetaService.getTag(dictConfDO.getItemId(), User.getFakeUser()); + dictItemResp.setModelId(tagResp.getModelId()); + dictItemResp.setBizName(tagResp.getBizName()); + } return dictItemResp; } public List fetchItemValue(DictItemResp dictItemResp) { List lines = new ArrayList<>(); - QuerySqlReq querySqlReq = constructQueryReq(dictItemResp); - querySqlReq.setNeedAuth(false); + SemanticQueryReq semanticQueryReq = constructQueryReq(dictItemResp); + semanticQueryReq.setNeedAuth(false); String bizName = dictItemResp.getBizName(); try { - SemanticQueryResp semanticQueryResp = queryService.queryByReq(querySqlReq, null); + SemanticQueryResp semanticQueryResp = queryService.queryByReq(semanticQueryReq, null); if (Objects.isNull(semanticQueryResp) || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { return lines; } @@ -173,7 +188,7 @@ public class DictUtils { mergeMultivaluedValue(valueAndFrequencyPair, dimValue, metric); } } - String nature = dictItemResp.getNature(); + String nature = dictItemResp.generateNature(); constructDictLines(valueAndFrequencyPair, lines, nature); addWhiteValueLines(dictItemResp, lines, nature); } catch (Exception e) { @@ -227,7 +242,51 @@ public class DictUtils { } } - private QuerySqlReq constructQueryReq(DictItemResp dictItemResp) { + private SemanticQueryReq constructQueryReq(DictItemResp dictItemResp) { + if (TypeEnums.DIMENSION.equals(dictItemResp.getType())) { + QuerySqlReq querySqlReq = constructDimQueryReq(dictItemResp); + querySqlReq.setQueryType(QueryType.METRIC); + return querySqlReq; + } + if (TypeEnums.TAG.equals(dictItemResp.getType())) { + QuerySqlReq querySqlReq = constructTagQueryReq(dictItemResp); + querySqlReq.setQueryType(QueryType.TAG); + return querySqlReq; + } + log.warn("constructQueryReq failed"); + return null; + } + + private QuerySqlReq constructTagQueryReq(DictItemResp dictItemResp) { + + String sqlPattern = "select %s, %s from tbl %s group by %s order by %s desc limit %d"; + String bizName = dictItemResp.getBizName(); + String whereStr = generateWhereStr(dictItemResp); + String where = Strings.isNullOrEmpty(whereStr) ? "" : "WHERE" + whereStr; + ItemValueConfig config = dictItemResp.getConfig(); + Long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount : + dictItemResp.getConfig().getLimit(); + + // todo 自定义指标 + String metric = "count(1)"; + if (Objects.nonNull(dictItemResp.getConfig()) && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { + Long metricId = dictItemResp.getConfig().getMetricId(); + MetricResp metricResp = metricService.getMetric(metricId); + String metricBizName = metricResp.getBizName(); + metric = String.format("sum(%s)", metricBizName); + } + + String sql = String.format(sqlPattern, bizName, metric, where, bizName, metric, limit); + Set modelIds = new HashSet<>(); + modelIds.add(dictItemResp.getModelId()); + QuerySqlReq querySqlReq = new QuerySqlReq(); + querySqlReq.setSql(sql); + querySqlReq.setNeedAuth(false); + querySqlReq.setModelIds(modelIds); + return querySqlReq; + } + + private QuerySqlReq constructDimQueryReq(DictItemResp dictItemResp) { if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig()) && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { // 查询默认指标 @@ -239,7 +298,6 @@ public class DictUtils { } private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { - // todo tag String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; String bizName = dictItemResp.getBizName(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java index 0c4d21f1f..3d9384fce 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java @@ -2,13 +2,19 @@ package com.tencent.supersonic.headless; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; import com.tencent.supersonic.headless.api.pojo.TagDefineParams; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; +import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; +import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq; import com.tencent.supersonic.headless.api.pojo.request.TagReq; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.server.pojo.TagFilter; +import com.tencent.supersonic.headless.server.service.DictConfService; +import com.tencent.supersonic.headless.server.service.DictTaskService; import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.service.TagQueryService; import org.junit.Assert; @@ -28,6 +34,10 @@ public class TagTest extends BaseTest { private TagMetaService tagMetaService; @Autowired private TagQueryService tagQueryService; + @Autowired + private DictConfService dictConfService; + @Autowired + private DictTaskService dictTaskService; @Test void testCreateTag() { @@ -95,4 +105,27 @@ public class TagTest extends BaseTest { tagMetaService.delete(tag.getId(), User.getFakeUser()); } + @Test + void testTagDict() { + User user = User.getFakeUser(); + TagReq tagReq = newTagReq(); + TagResp tagResp = tagMetaService.create(tagReq, user); + // add conf + DictItemReq itemValueReq = new DictItemReq(); + itemValueReq.setType(TypeEnums.TAG); + itemValueReq.setItemId(tagResp.getId()); + itemValueReq.setStatus(StatusEnum.ONLINE); + ItemValueConfig config = new ItemValueConfig(); + config.setMetricId(4L); + config.setWhiteList(Arrays.asList("p10", "p20")); + config.setBlackList(Arrays.asList("p1", "p2")); + itemValueReq.setConfig(config); + dictConfService.addDictConf(itemValueReq, user); + // run Task + DictSingleTaskReq taskReq = DictSingleTaskReq.builder().type(TypeEnums.TAG).itemId(tagResp.getId()).build(); + dictTaskService.addDictTask(taskReq, user); + + tagMetaService.delete(tagResp.getId(), user); + } + } \ No newline at end of file