From 97bf8049d73022ed165a09ebe7b84bb62b824728 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Sat, 3 Aug 2024 23:56:29 +0800 Subject: [PATCH] (improvement)(headless) Divide dimensions into four types: identify, category, time and partition time (#1509) Co-authored-by: lxwcodemonkey --- .../common/pojo/DimensionConstants.java | 9 ++++++++ .../headless/api/pojo/SchemaElement.java | 6 +++-- .../api/pojo/enums/DimensionType.java | 12 +++++++++- .../api/pojo/response/DimensionResp.java | 7 +++++- .../server/manager/DimensionYamlManager.java | 2 +- .../server/utils/DataSetSchemaBuilder.java | 11 +++++---- .../server/utils/DimensionConverter.java | 2 ++ .../headless/server/utils/ModelConverter.java | 23 ++++++++++--------- .../mapper/custom/DimensionDOCustomMapper.xml | 5 ++-- 9 files changed, 55 insertions(+), 22 deletions(-) create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java new file mode 100644 index 000000000..346ff00d9 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java @@ -0,0 +1,9 @@ +package com.tencent.supersonic.common.pojo; + +public class DimensionConstants { + + public static final String DIMENSION_TIME_FORMAT = "time_format"; + + public static final String DIMENSION_TYPE = "dimension_type"; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index a0c48e1e7..8022c0fa2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.base.Objects; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -9,7 +8,9 @@ import lombok.Getter; import lombok.NoArgsConstructor; import java.io.Serializable; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Data @Getter @@ -26,7 +27,6 @@ public class SchemaElement implements Serializable { private String bizName; private Long useCnt; private SchemaElementType type; - private SemanticType semanticType; private List alias; private List schemaValueMaps; private List relatedSchemaElements; @@ -36,6 +36,8 @@ public class SchemaElement implements Serializable { private int isTag; private String description; private boolean descriptionMapped; + @Builder.Default + private Map extInfo = new HashMap<>(); @Override public boolean equals(Object o) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java index b23ee68ab..355e83536 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java @@ -4,6 +4,16 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum DimensionType { categorical, - time + time, + partition_time, + identify; + + public static Boolean isTimeDimension(String type) { + return time.name().equals(type) || partition_time.name().equals(type); + } + + public static Boolean isTimeDimension(DimensionType type) { + return time.equals(type) || partition_time.equals(type); + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java index fb29871a4..09dd4966d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java @@ -4,6 +4,7 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import lombok.Data; import lombok.ToString; @@ -18,7 +19,7 @@ public class DimensionResp extends SchemaItem { private Long modelId; - private String type; + private DimensionType type; private String expr; @@ -42,4 +43,8 @@ public class DimensionResp extends SchemaItem { private Map ext = new HashMap<>(); + public boolean isTimeDimension() { + return DimensionType.isTimeDimension(type); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java index 24940bb0c..de91070a6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java @@ -25,7 +25,7 @@ public class DimensionYamlManager { return new ArrayList<>(); } return dimensions.stream() - .filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name())) + .filter(dimension -> !dimension.getType().name().equalsIgnoreCase(IdentifyType.primary.name())) .map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 44a5a90ba..f76ffc055 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.DimensionConstants; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.RelateDimension; @@ -9,7 +10,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; @@ -155,7 +155,6 @@ public class DataSetSchemaBuilder { schemaValueMaps.add(schemaValueMap); } } - SemanticType semanticType = SemanticType.valueOf(dim.getSemanticType()); SchemaElement dimToAdd = SchemaElement.builder() .dataSetId(resp.getId()) .dataSetName(resp.getName()) @@ -163,14 +162,18 @@ public class DataSetSchemaBuilder { .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) - .type(SchemaElementType.DIMENSION) - .semanticType(semanticType) .useCnt(dim.getUseCnt()) .alias(alias) .schemaValueMaps(schemaValueMaps) .isTag(dim.getIsTag()) .description(dim.getDescription()) + .type(SchemaElementType.DIMENSION) .build(); + dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType()); + if (dim.isTimeDimension()) { + String timeFormat = String.valueOf(dim.getExt().get(DimensionConstants.DIMENSION_TIME_FORMAT)); + dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat); + } dimensions.add(dimToAdd); } return dimensions; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 8b74aee9e..1b396b53a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; @@ -85,6 +86,7 @@ public class DimensionConverter { if (dimensionDO.getExt() != null) { dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class)); } + dimensionResp.setType(DimensionType.valueOf(dimensionDO.getType())); dimensionResp.setTypeEnum(TypeEnums.DIMENSION); return dimensionResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index 1ffa5e581..69fc0623f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.server.utils; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DimensionConstants; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Identify; @@ -16,6 +14,9 @@ import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; @@ -29,6 +30,7 @@ import org.springframework.util.CollectionUtils; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -108,14 +110,17 @@ public class ModelConverter { dimensionReq.setName(dim.getName()); dimensionReq.setBizName(dim.getBizName()); dimensionReq.setDescription(dim.getName()); - if (Objects.equals(dim.getType(), DimensionType.time.name())) { + if (DimensionType.isTimeDimension(dim.getType())) { dimensionReq.setSemanticType(SemanticType.DATE.name()); + Map map = new HashMap<>(); + map.put(DimensionConstants.DIMENSION_TIME_FORMAT, dim.getDateFormat()); + dimensionReq.setExt(map); } else { dimensionReq.setSemanticType(SemanticType.CATEGORY.name()); } dimensionReq.setModelId(modelDO.getId()); dimensionReq.setExpr(dim.getBizName()); - dimensionReq.setType(DimensionType.categorical.name()); + dimensionReq.setType(dim.getType()); dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); dimensionReq.setIsTag(dim.getIsTag()); return dimensionReq; @@ -142,14 +147,10 @@ public class ModelConverter { dimensionReq.setName(identify.getName()); dimensionReq.setBizName(identify.getBizName()); dimensionReq.setDescription(identify.getName()); - if (Objects.equals(identify.getType(), DimensionType.time.name())) { - dimensionReq.setSemanticType(SemanticType.DATE.name()); - } else { - dimensionReq.setSemanticType(SemanticType.CATEGORY.name()); - } + dimensionReq.setSemanticType(SemanticType.CATEGORY.name()); dimensionReq.setModelId(modelDO.getId()); dimensionReq.setExpr(identify.getBizName()); - dimensionReq.setType(identify.getType()); + dimensionReq.setType(DimensionType.identify.name()); return dimensionReq; } diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index 95b9f734d..5629a2f63 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -67,7 +67,7 @@ insert into s2_dimension (name, biz_name, description, status, model_id, type, type_params, expr,created_at, created_by, - updated_by, updated_at, semantic_type,sensitive_level, is_tag) + updated_by, updated_at, semantic_type,sensitive_level, is_tag, ext) values (#{dimension.name,jdbcType=VARCHAR}, #{dimension.bizName,jdbcType=VARCHAR}, @@ -78,7 +78,8 @@ #{dimension.createdBy,jdbcType=VARCHAR}, #{dimension.updatedBy,jdbcType=VARCHAR}, #{dimension.updatedAt,jdbcType=TIMESTAMP}, #{dimension.semanticType,jdbcType=VARCHAR}, - #{dimension.sensitiveLevel,jdbcType=INTEGER}, #{dimension.isTag, jdbcType=INTEGER}) + #{dimension.sensitiveLevel,jdbcType=INTEGER}, #{dimension.isTag, jdbcType=INTEGER}, + #{dimension.ext, jdbcType=VARCHAR})