topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(semantic) add materialization optimizer (#239)

Browse Source 
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-10-16 22:07:45 +08:00
committed by GitHub
parent 5b8fde70ca
commit 40ba179703
126 changed files with 9172 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -228,8 +228,9 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<Long, String> getModelFullPathMap() {
return getModelList().stream().filter(Objects::nonNull).collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
return getModelList().stream().filter(m -> m != null && m.getFullPath() != null)
.collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
}
@Override

8
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java
View File

@@ -58,6 +58,8 @@ public class DatasourceDO {
*/
private String datasourceDetail;
private String depends;
/**
* @return id
@@ -253,5 +255,11 @@ public class DatasourceDO {
this.datasourceDetail = datasourceDetail == null ? null : datasourceDetail.trim();
}
public void setDepends(String depends) {
this.depends = depends;
}
public String getDepends() {
return depends;
}
}

15
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DimensionDO.java
View File

@@ -98,6 +98,11 @@ public class DimensionDO {
*/
private String expr;
/**
* 数据类型
*/
private String dataType;
/**
* 维度ID
*
@@ -431,4 +436,12 @@ public class DimensionDO {
public void setExpr(String expr) {
this.expr = expr == null ? null : expr.trim();
}
}
public String getDataType() {
return dataType;
}
public void setDataType(String dataType) {
this.dataType = dataType;
}
}

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Dimension.java
View File

@@ -1,9 +1,11 @@
package com.tencent.supersonic.semantic.model.domain.pojo;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import lombok.Data;
import java.util.List;
@Data
@@ -26,4 +28,6 @@ public class Dimension extends SchemaItem {
private List<DimValueMap> dimValueMaps;
private DataTypeEnums dataType;
}

6
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java
View File

@@ -3,6 +3,8 @@ package com.tencent.supersonic.semantic.model.domain.utils;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.semantic.api.model.enums.MetricTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
@@ -15,13 +17,12 @@ import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.response.DatasourceRelaResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceRelaDO;
import com.tencent.supersonic.semantic.model.domain.pojo.Datasource;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -108,6 +109,7 @@ public class DatasourceConverter {
dimensionReq.setModelId(datasource.getModelId());
dimensionReq.setExpr(dim.getBizName());
dimensionReq.setType("categorical");
dimensionReq.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
return dimensionReq;
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.model.domain.utils;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -14,6 +15,7 @@ import com.tencent.supersonic.semantic.model.domain.pojo.Dimension;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -35,6 +37,9 @@ public class DimensionConverter {
} else {
dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>()));
}
if (Objects.nonNull(dimension.getDataType())) {
dimensionDO.setDataType(dimension.getDataType().getType());
}
return dimensionDO;
}
@@ -43,6 +48,9 @@ public class DimensionConverter {
BeanUtils.copyProperties(dimension, dimensionDO);
dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues()));
dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps()));
if (Objects.nonNull(dimension.getDataType())) {
dimensionDO.setDataType(dimension.getDataType().getType());
}
return dimensionDO;
}
@@ -65,6 +73,9 @@ public class DimensionConverter {
if (Strings.isNotEmpty(dimensionDO.getDimValueMaps())) {
dimensionResp.setDimValueMaps(JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class));
}
if (Strings.isNotEmpty(dimensionDO.getDataType())) {
dimensionResp.setDataType(DataTypeEnums.of(dimensionDO.getDataType()));
}
return dimensionResp;
}

7
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java
View File

@@ -77,7 +77,12 @@ public class MetricConverter {
RelateDimension.class));
return metricResp;
}
public static Metric convert2Metric(MetricDO metricDO) {
Metric metric = new Metric();
BeanUtils.copyProperties(metricDO, metric);
metric.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class));
return metric;
}
public static MetricYamlTpl convert2MetricYamlTpl(Metric metric) {
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();

3
semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml
View File

@@ -12,6 +12,7 @@
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
<result column="depends" jdbcType="VARCHAR" property="depends" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO">
<result column="datasource_detail" jdbcType="LONGVARCHAR" property="datasourceDetail" />
@@ -47,7 +48,7 @@
</sql>
<sql id="Base_Column_List">
id, model_id, name, biz_name, description, database_id, created_at, created_by, updated_at,
updated_by
updated_by,depends
</sql>
<sql id="Blob_Column_List">
datasource_detail

24
semantic/model/src/main/resources/mapper/DimensionDOMapper.xml
View File

@@ -19,6 +19,7 @@
<result column="alias" jdbcType="VARCHAR" property="alias" />
<result column="default_values" jdbcType="VARCHAR" property="defaultValues" />
<result column="dim_value_maps" jdbcType="VARCHAR" property="dimValueMaps" />
<result column="data_type" jdbcType="VARCHAR" property="dataType"/>
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO">
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" />
@@ -56,7 +57,7 @@
<sql id="Base_Column_List">
id, model_id, datasource_id, name, biz_name, description, status, sensitive_level,
type, created_at, created_by, updated_at, updated_by, semantic_type, alias, default_values,
dim_value_maps
dim_value_maps, data_type
</sql>
<sql id="Blob_Column_List">
type_params, expr
@@ -113,14 +114,14 @@
created_at, created_by, updated_at,
updated_by, semantic_type, alias,
default_values, dim_value_maps, type_params,
expr)
expr, data_type)
values (#{id,jdbcType=BIGINT}, #{modelId,jdbcType=BIGINT}, #{datasourceId,jdbcType=BIGINT},
#{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR},
#{status,jdbcType=INTEGER}, #{sensitiveLevel,jdbcType=INTEGER}, #{type,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{semanticType,jdbcType=VARCHAR}, #{alias,jdbcType=VARCHAR},
#{defaultValues,jdbcType=VARCHAR}, #{dimValueMaps,jdbcType=VARCHAR}, #{typeParams,jdbcType=LONGVARCHAR},
#{expr,jdbcType=LONGVARCHAR})
#{defaultValues,jdbcType=VARCHAR}, #{dimValueMaps,jdbcType=VARCHAR}, #{typeParams,jdbcType=LONGVARCHAR},
#{expr,jdbcType=LONGVARCHAR}, #{dataType,jdbcType=VARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO">
insert into s2_dimension
@@ -182,6 +183,9 @@
<if test="expr != null">
expr,
</if>
<if test="data_type != null">
data_type,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
@@ -241,6 +245,9 @@
<if test="expr != null">
#{expr,jdbcType=LONGVARCHAR},
</if>
<if test="dataType != null">
#{dataType,jdbcType=VARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDOExample" resultType="java.lang.Long">
@@ -306,6 +313,9 @@
<if test="expr != null">
expr = #{expr,jdbcType=LONGVARCHAR},
</if>
<if test="dataType != null">
data_type = #{dataType,jdbcType=VARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
@@ -328,7 +338,8 @@
default_values = #{defaultValues,jdbcType=VARCHAR},
dim_value_maps = #{dimValueMaps,jdbcType=VARCHAR},
type_params = #{typeParams,jdbcType=LONGVARCHAR},
expr = #{expr,jdbcType=LONGVARCHAR}
expr = #{expr,jdbcType=LONGVARCHAR},
data_type = #{dataType,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO">
@@ -348,7 +359,8 @@
semantic_type = #{semanticType,jdbcType=VARCHAR},
alias = #{alias,jdbcType=VARCHAR},
default_values = #{defaultValues,jdbcType=VARCHAR},
dim_value_maps = #{dimValueMaps,jdbcType=VARCHAR}
dim_value_maps = #{dimValueMaps,jdbcType=VARCHAR},
data_type = #{dataType,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>