(improvement)(headless) Divide dimensions into four types: identify, category, time and partition time (#1509)

Co-authored-by: lxwcodemonkey

common/src/main/java/com/tencent/supersonic/common/pojo/DimensionConstants.java

@@ -0,0 +1,9 @@
+package com.tencent.supersonic.common.pojo;
+
+public class DimensionConstants {
+
+    public static final String DIMENSION_TIME_FORMAT = "time_format";
+
+    public static final String DIMENSION_TYPE = "dimension_type";
+
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo;
 
 import com.google.common.base.Objects;
-import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -9,7 +8,9 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 @Data
 @Getter
@@ -26,7 +27,6 @@ public class SchemaElement implements Serializable {
     private String bizName;
     private Long useCnt;
     private SchemaElementType type;
-    private SemanticType semanticType;
     private List<String> alias;
     private List<SchemaValueMap> schemaValueMaps;
     private List<RelatedSchemaElement> relatedSchemaElements;
@@ -36,6 +36,8 @@ public class SchemaElement implements Serializable {
     private int isTag;
     private String description;
     private boolean descriptionMapped;
+    @Builder.Default
+    private Map<String, Object> extInfo = new HashMap<>();
 
     @Override
     public boolean equals(Object o) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DimensionType.java

@@ -4,6 +4,16 @@ package com.tencent.supersonic.headless.api.pojo.enums;
 public enum DimensionType {
 
     categorical,
-    time
+    time,
+    partition_time,
+    identify;
+
+    public static Boolean isTimeDimension(String type) {
+        return time.name().equals(type) || partition_time.name().equals(type);
+    }
+
+    public static Boolean isTimeDimension(DimensionType type) {
+        return time.equals(type) || partition_time.equals(type);
+    }
 
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -4,6 +4,7 @@ package com.tencent.supersonic.headless.api.pojo.response;
 import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import lombok.Data;
 import lombok.ToString;
 
@@ -18,7 +19,7 @@ public class DimensionResp extends SchemaItem {
 
     private Long modelId;
 
-    private String type;
+    private DimensionType type;
 
     private String expr;
 
@@ -42,4 +43,8 @@ public class DimensionResp extends SchemaItem {
 
     private Map<String, Object> ext = new HashMap<>();
 
+    public boolean isTimeDimension() {
+        return DimensionType.isTimeDimension(type);
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java

@@ -25,7 +25,7 @@ public class DimensionYamlManager {
             return new ArrayList<>();
         }
         return dimensions.stream()
-                .filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name()))
+                .filter(dimension -> !dimension.getType().name().equalsIgnoreCase(IdentifyType.primary.name()))
                 .map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.server.utils;
 
 import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.DimensionConstants;
 import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
@@ -9,7 +10,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
-import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -155,7 +155,6 @@ public class DataSetSchemaBuilder {
                     schemaValueMaps.add(schemaValueMap);
                 }
             }
-            SemanticType semanticType = SemanticType.valueOf(dim.getSemanticType());
             SchemaElement dimToAdd = SchemaElement.builder()
                     .dataSetId(resp.getId())
                     .dataSetName(resp.getName())
@@ -163,14 +162,18 @@ public class DataSetSchemaBuilder {
                     .id(dim.getId())
                     .name(dim.getName())
                     .bizName(dim.getBizName())
-                    .type(SchemaElementType.DIMENSION)
-                    .semanticType(semanticType)
                     .useCnt(dim.getUseCnt())
                     .alias(alias)
                     .schemaValueMaps(schemaValueMaps)
                     .isTag(dim.getIsTag())
                     .description(dim.getDescription())
+                    .type(SchemaElementType.DIMENSION)
                     .build();
+            dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType());
+            if (dim.isTimeDimension()) {
+                String timeFormat = String.valueOf(dim.getExt().get(DimensionConstants.DIMENSION_TIME_FORMAT));
+                dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
+            }
             dimensions.add(dimToAdd);
         }
         return dimensions;

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
+import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
 import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
@@ -85,6 +86,7 @@ public class DimensionConverter {
         if (dimensionDO.getExt() != null) {
             dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
         }
+        dimensionResp.setType(DimensionType.valueOf(dimensionDO.getType()));
         dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
         return dimensionResp;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.server.utils;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.DimensionConstants;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
-import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
-import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
 import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
 import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -16,6 +14,9 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
+import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
+import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
@@ -29,6 +30,7 @@ import org.springframework.util.CollectionUtils;
 
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -108,14 +110,17 @@ public class ModelConverter {
         dimensionReq.setName(dim.getName());
         dimensionReq.setBizName(dim.getBizName());
         dimensionReq.setDescription(dim.getName());
-        if (Objects.equals(dim.getType(), DimensionType.time.name())) {
+        if (DimensionType.isTimeDimension(dim.getType())) {
             dimensionReq.setSemanticType(SemanticType.DATE.name());
+            Map<String, Object> map = new HashMap<>();
+            map.put(DimensionConstants.DIMENSION_TIME_FORMAT, dim.getDateFormat());
+            dimensionReq.setExt(map);
         } else {
             dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
         }
         dimensionReq.setModelId(modelDO.getId());
         dimensionReq.setExpr(dim.getBizName());
-        dimensionReq.setType(DimensionType.categorical.name());
+        dimensionReq.setType(dim.getType());
         dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
         dimensionReq.setIsTag(dim.getIsTag());
         return dimensionReq;
@@ -142,14 +147,10 @@ public class ModelConverter {
         dimensionReq.setName(identify.getName());
         dimensionReq.setBizName(identify.getBizName());
         dimensionReq.setDescription(identify.getName());
-        if (Objects.equals(identify.getType(), DimensionType.time.name())) {
+        dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
-            dimensionReq.setSemanticType(SemanticType.DATE.name());
-        } else {
-            dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
-        }
         dimensionReq.setModelId(modelDO.getId());
         dimensionReq.setExpr(identify.getBizName());
-        dimensionReq.setType(identify.getType());
+        dimensionReq.setType(DimensionType.identify.name());
         return dimensionReq;
     }
 

headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml

@@ -67,7 +67,7 @@
         insert into s2_dimension (name, biz_name,
         description, status, model_id,
         type, type_params, expr,created_at, created_by,
-        updated_by, updated_at, semantic_type,sensitive_level, is_tag)
+        updated_by, updated_at, semantic_type,sensitive_level, is_tag, ext)
         values
         <foreach collection="list" item="dimension" separator=",">
             (#{dimension.name,jdbcType=VARCHAR}, #{dimension.bizName,jdbcType=VARCHAR},
@@ -78,7 +78,8 @@
             #{dimension.createdBy,jdbcType=VARCHAR},
             #{dimension.updatedBy,jdbcType=VARCHAR}, #{dimension.updatedAt,jdbcType=TIMESTAMP},
             #{dimension.semanticType,jdbcType=VARCHAR},
-            #{dimension.sensitiveLevel,jdbcType=INTEGER}, #{dimension.isTag, jdbcType=INTEGER})
+            #{dimension.sensitiveLevel,jdbcType=INTEGER}, #{dimension.isTag, jdbcType=INTEGER},
+            #{dimension.ext, jdbcType=VARCHAR})
         </foreach>
     </insert>