From 9cd352f146e25acf6d291e0eb1c33a57cba5b52c Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Tue, 2 Jul 2024 18:46:05 +0800 Subject: [PATCH] Add dict date (#1331) --- .../supersonic/common/pojo/DateConf.java | 3 +- .../headless/api/pojo/ItemValueConfig.java | 2 + .../headless/server/utils/DictUtils.java | 121 ++++++++++++++++-- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java index a3516c643..17cc06115 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java @@ -98,8 +98,9 @@ public class DateConf { * 2 - LIST, discrete static value, [dateList] * 3 - RECENT, dynamic time related to the actual available time of the element, [unit, period] * 4 - AVAILABLE, dynamic time which guaranteed to query some data, [startDate, endDate] + * 5 - ALL, all table data */ - BETWEEN, LIST, RECENT, AVAILABLE + BETWEEN, LIST, RECENT, AVAILABLE, ALL } @Override diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java index c108012cf..54c48785b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.api.pojo; +import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; import java.util.ArrayList; @@ -17,4 +18,5 @@ public class ItemValueConfig { private List whiteList = new ArrayList<>(); private List ruleList = new ArrayList<>(); private Long limit; + private DateConf dateConf; } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 488b3ecf0..707ae026b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.ItemValueConfig; @@ -35,6 +36,7 @@ import com.tencent.supersonic.headless.server.web.service.ModelService; import com.tencent.supersonic.headless.server.web.service.TagMetaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -72,11 +74,16 @@ public class DictUtils { @Value("${s2.item.value.white.frequency:999999}") private Long itemValueWhiteFrequency; + // 前多少天 @Value("${s2.item.value.date.start:1}") private Integer itemValueDateStart; @Value("${s2.item.value.date.end:1}") + // 前多少天 private Integer itemValueDateEnd; + @Value("${s2.item.value.date.format:yyyy-MM-dd}") + private String itemValueDateFormat; + private final DimensionService dimensionService; private final MetricService metricService; @@ -85,10 +92,10 @@ public class DictUtils { private final TagMetaService tagMetaService; public DictUtils(DimensionService dimensionService, - MetricService metricService, - SemanticLayerService queryService, - ModelService modelService, - @Lazy TagMetaService tagMetaService) { + MetricService metricService, + SemanticLayerService queryService, + ModelService modelService, + @Lazy TagMetaService tagMetaService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; @@ -131,7 +138,13 @@ public class DictUtils { public List dictDOList2Req(List dictConfDOList) { List dictItemReqList = new ArrayList<>(); - dictConfDOList.stream().forEach(conf -> dictItemReqList.add(dictDO2Req(conf))); + dictConfDOList.stream().forEach(conf -> { + DictItemResp dictItemResp = dictDO2Req(conf); + if (Objects.nonNull(dictItemResp)) { + dictItemReqList.add(dictDO2Req(conf)); + } + + }); return dictItemReqList; } @@ -143,6 +156,10 @@ public class DictUtils { dictItemResp.setStatus(StatusEnum.of(dictConfDO.getStatus())); if (TypeEnums.DIMENSION.equals(TypeEnums.valueOf(dictConfDO.getType()))) { DimensionResp dimension = dimensionService.getDimension(dictConfDO.getItemId()); + if (Objects.isNull(dimension)) { + log.info("dimension is null, dictConfDO:{}", dictConfDO); + return null; + } dictItemResp.setModelId(dimension.getModelId()); dictItemResp.setBizName(dimension.getBizName()); } @@ -340,7 +357,38 @@ public class DictUtils { } private void fillStructDateInfo(QueryStructReq queryStructReq, DictItemResp dictItemResp) { + + ItemValueConfig config = dictItemResp.getConfig(); + ModelResp model = modelService.getModel(dictItemResp.getModelId()); + // 用户未进行设置 + if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) { + fillStructDateBetween(queryStructReq, model, itemValueDateStart, itemValueDateEnd); + return; + } + + // 全表扫描 + if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) { + return; + } + // 静态日期 + if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) { + DateConf dateConf = new DateConf(); + BeanMapper.mapper(config.getDateConf(), dateConf); + dateConf.setDateMode(DateConf.DateMode.BETWEEN); + queryStructReq.setDateInfo(dateConf); + return; + } + // 动态日期 包括今天日期 + if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) { + fillStructDateBetween(queryStructReq, model, config.getDateConf().getUnit() - 1, 0); + return; + } + return; + } + + private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model, + Integer itemValueDateStart, Integer itemValueDateEnd) { if (Objects.nonNull(model)) { List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { @@ -389,20 +437,67 @@ public class DictUtils { } } + String dateFilter = generateDictDateFilter(dictItemResp); + if (Strings.isNotEmpty(dateFilter)) { + joiner.add(dateFilter); + } + return joiner.toString(); + } + + public String defaultDateFilter() { + String format = itemValueDateFormat; + String start = LocalDate.now().minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(format)); + String end = LocalDate.now().minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(format)); + return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, + TimeDimensionEnum.DAY.getName(), end); + } + + private String generateDictDateFilter(DictItemResp dictItemResp) { + ItemValueConfig config = dictItemResp.getConfig(); + // 未进行设置 + if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) { + return defaultDateFilter(); + } + // 全表扫描 + if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) { + return ""; + } + // 静态日期 + if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) { + return String.format("( %s >= '%s' and %s <= '%s' )", + TimeDimensionEnum.DAY.getName(), + config.getDateConf().getStartDate(), + TimeDimensionEnum.DAY.getName(), + config.getDateConf().getEndDate()); + } + // 动态日期 + if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) { + return generateDictDateFilterRecent(dictItemResp); + } + + return ""; + } + + private String generateDictDateFilterRecent(DictItemResp dictItemResp) { ModelResp model = modelService.getModel(dictItemResp.getModelId()); if (Objects.nonNull(model)) { List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { - String format = "yyyy-MM-dd"; - String start = LocalDate.now().minusDays(itemValueDateStart) - .format(DateTimeFormatter.ofPattern(format)); - String end = LocalDate.now().minusDays(itemValueDateEnd) - .format(DateTimeFormatter.ofPattern(format)); - joiner.add(String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, - TimeDimensionEnum.DAY.getName(), end)); + String dateFormat = timeDims.get(0).getDateFormat(); + if (Strings.isEmpty(dateFormat)) { + dateFormat = itemValueDateFormat; + } + String start = LocalDate.now().minusDays(dictItemResp.getConfig().getDateConf().getUnit()) + .format(DateTimeFormatter.ofPattern(dateFormat)); + String end = LocalDate.now().minusDays(0) + .format(DateTimeFormatter.ofPattern(dateFormat)); + return String.format("( %s > '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), start, + TimeDimensionEnum.DAY.getName(), end); } } - return joiner.toString(); + return ""; } public DictTaskResp taskDO2Resp(DictTaskDO dictTaskDO) {