topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(semantic) add materialization optimizer (#239)

Browse Source 
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-10-16 22:07:45 +08:00
committed by GitHub
parent 5b8fde70ca
commit 40ba179703
126 changed files with 9172 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/enums/ElementFrequencyEnum.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.semantic.api.materialization.enums;
public enum ElementFrequencyEnum {
UNKNOWN,
HIGH,
LOW
}

11
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/enums/ElementTypeEnum.java Normal file
View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.semantic.api.materialization.enums;
public enum ElementTypeEnum {
VARCHAR,
DOUBLE,
BIGINT,
INT,
DATE,
ARRAY
}

7
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/enums/MaterializedTypeEnum.java Normal file
View File

@@ -0,0 +1,7 @@
package com.tencent.supersonic.semantic.api.materialization.enums;
public enum MaterializedTypeEnum {
FULL,
PARTITION,
ZIPPER
}

8
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/enums/UpdateCycleEnum.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.semantic.api.materialization.enums;
public enum UpdateCycleEnum {
DAY,
WEEK,
MONTH
}

17
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/pojo/MaterializationConfFilter.java Normal file
View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.semantic.api.materialization.pojo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class MaterializationConfFilter extends MaterializationFilter {
private Long id;
private Boolean containElements = false;
private TypeEnums type;
private Long materializationId;
private Long elementId;
}

16
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/pojo/MaterializationDateFilter.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.semantic.api.materialization.pojo;
import lombok.Data;
import java.util.Date;
import java.util.List;
@Data
public class MaterializationDateFilter {
private Long modelId;
private List<String> dimensions;
private List<String> metrics;
private Date createdAt;
}

25
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/pojo/MaterializationFilter.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.semantic.api.materialization.pojo;
import com.tencent.supersonic.semantic.api.materialization.enums.MaterializedTypeEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.UpdateCycleEnum;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MaterializationFilter {
private Long materializationId;
private String name;
private MaterializedTypeEnum materializedType;
private UpdateCycleEnum updateCycle;
private Long modelId;
private Long databaseId;
private Integer level;
private String createdBy;
private String destinationTable;
}

28
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/pojo/MaterializationRecordFilter.java Normal file
View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.semantic.api.materialization.pojo;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.List;
import lombok.Builder;
import lombok.Data;
import java.util.Date;
@Data
@Builder
public class MaterializationRecordFilter {
private Long id;
private Long materializationId;
private TypeEnums elementType;
private Long elementId;
private String elementName;
private List<TaskStatusEnum> taskStatus;
private String taskId;
private String createdBy;
private Date createdAt;
private String startDataTime;
private String endDataTime;
private List<Long> materializationIds;
}

25
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/request/MaterializationElementReq.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.semantic.api.materialization.request;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementTypeEnum;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MaterializationElementReq extends RecordInfo {
private Long id;
private TypeEnums type;
private Long materializationId;
private String depends;
private ElementTypeEnum elementType;
private String defaultValue;
private String outlier;
private ElementFrequencyEnum frequency;
private String description;
private StatusEnum status;
}

24
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/request/MaterializationRecordReq.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.semantic.api.materialization.request;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Data;
@Data
public class MaterializationRecordReq extends RecordInfo {
private Long id;
private Long materializationId;
private TypeEnums elementType;
private Long elementId;
private String elementName;
private String dataTime;
private TaskStatusEnum taskStatus;
private String taskId;
private Long retryCount;
private Long sourceCount;
private Long sinkCount;
private String message;
}

26
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/request/MaterializationReq.java Normal file
View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.semantic.api.materialization.request;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.MaterializedTypeEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.UpdateCycleEnum;
import lombok.Data;
import java.util.List;
@Data
public class MaterializationReq extends RecordInfo {
private Long id;
private String name;
private MaterializedTypeEnum materializedType;
private UpdateCycleEnum updateCycle;
private Long modelId;
private Long databaseId;
private Integer level;
private String destinationTable;
private String dateInfo;
private String entities;
private List<String> principals;
private String description;
private StatusEnum status;
}

12
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/request/MaterializationSourceReq.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.semantic.api.materialization.request;
import lombok.Data;
@Data
public class MaterializationSourceReq {
private Long materializationId = 0L;
private Long dataSourceId = 0L;
private String dataTime;
}

20
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationDateResp.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class MaterializationDateResp {
private Long modelId;
private TypeEnums elementType;
private Long elementId;
private String elementName;
private String dateFormat;
private String startDate;
private String endDate;
private List<String> unavailableDateList = new ArrayList<>();
}

18
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationElementModelResp.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import java.util.List;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MaterializationElementModelResp {
private Long id;
private TypeEnums type;
private String bizName;
private String expr;
private List<Measure> measures = Lists.newArrayList();
}

23
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationElementResp.java Normal file
View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementTypeEnum;
import lombok.Data;
@Data
public class MaterializationElementResp extends RecordInfo {
private Long id;
private TypeEnums type;
private Long materializationId;
private String depends;
private ElementTypeEnum elementType;
private String defaultValue;
private String outlier;
private ElementFrequencyEnum frequency;
private String description;
private String bizName;
}

24
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationRecordResp.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Data;
@Data
public class MaterializationRecordResp extends RecordInfo {
private Long id;
private Long materializationId;
private TypeEnums elementType;
private Long elementId;
private String elementName;
private String dataTime;
private TaskStatusEnum taskStatus;
private String taskId;
private Integer retryCount;
private Long sourceCount;
private Long sinkCount;
private String message;
}

25
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationResp.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.semantic.api.materialization.enums.MaterializedTypeEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.UpdateCycleEnum;
import lombok.Data;
import java.util.List;
@Data
public class MaterializationResp extends RecordInfo {
private Long id;
private String name;
private MaterializedTypeEnum materializedType;
private UpdateCycleEnum updateCycle;
private Long modelId;
private Long databaseId;
private Integer level;
private String destinationTable;
private String dateInfo;
private String entities;
private List<String> principals;
private String description;
private List<MaterializationElementResp> materializationElementRespList;
}

35
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/materialization/response/MaterializationSourceResp.java Normal file
View File

@@ -0,0 +1,35 @@
package com.tencent.supersonic.semantic.api.materialization.response;
import com.tencent.supersonic.semantic.api.materialization.enums.MaterializedTypeEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.UpdateCycleEnum;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class MaterializationSourceResp {
private Long materializationId;
private Long dataSourceId;
private Long modelId;
private String sql;
private List<String> fields;
private String sourceDb;
private String sourceTable;
private String dateInfo;
private String entities;
private MaterializedTypeEnum materializedType;
private UpdateCycleEnum updateCycle;
private DatabaseResp databaseResp;
private String depends;
private Map<Long, String> dimensions;
private Map<Long, String> metrics;
}

8
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/enums/QueryOptMode.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.semantic.api.model.enums;
public enum QueryOptMode {
NONE,
MATERIALIZATION
}

13
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/Dim.java
View File

@@ -25,6 +25,8 @@ public class Dim {
private String bizName;
private String description;
public Dim(String name, String bizName, String type, Integer isCreateDimension) {
this.name = name;
this.type = type;
@@ -32,6 +34,17 @@ public class Dim {
this.bizName = bizName;
}
public Dim(String name, String type, String expr, String dateFormat, DimensionTimeTypeParams typeParams,
Integer isCreateDimension, String bizName) {
this.name = name;
this.type = type;
this.expr = expr;
this.dateFormat = dateFormat;
this.typeParams = typeParams;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
}
public static Dim getDefault() {
return new Dim("日期", "time", "2023-05-28",
Constants.DAY_FORMAT,

8
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/QueryStat.java
View File

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.Objects;
import lombok.Data;
import lombok.ToString;
@@ -48,6 +49,13 @@ public class QueryStat {
private Boolean useSqlCache;
private String sqlCacheKey;
private String resultCacheKey;
private String queryOptMode;
public QueryStat setQueryOptMode(String queryOptMode) {
this.queryOptMode = queryOptMode;
return this;
}
public QueryStat setQuerySqlCmdMd5(String querySqlCmdMd5) {
this.querySqlCmdMd5 = querySqlCmdMd5;

3
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DimensionReq.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.api.model.request;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
@@ -30,4 +31,6 @@ public class DimensionReq extends SchemaItem {
private List<String> defaultValues;
private List<DimValueMap> dimValueMaps;
private DataTypeEnums dataType;
}

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DatasourceResp.java
View File

@@ -13,6 +13,8 @@ public class DatasourceResp extends SchemaItem {
private DatasourceDetail datasourceDetail;
private String depends;
}

3
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DimensionResp.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.api.model.response;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
@@ -36,4 +37,6 @@ public class DimensionResp extends SchemaItem {
private List<DimValueMap> dimValueMaps;
private DataTypeEnums dataType;
}