topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](materialization) materializationZipperUtils support array type (#289)

Browse Source
This commit is contained in:
daikon
2023-10-25 20:21:24 +08:00
committed by GitHub
parent d533496b2a
commit e44e7ca8d5
2 changed files with 39 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java
View File

@@ -30,10 +30,10 @@ import com.tencent.supersonic.semantic.materialization.domain.pojo.Materializati
import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationElementRepository;
import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationRepository;
import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationConverter;
import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationUtils;
import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationZipperUtils;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -44,6 +44,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -58,16 +59,18 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
private final MaterializationElementRepository materializationElementRepository;
private final ModelService modelService;
private final DatasourceService datasourceService;
private final MaterializationZipperUtils materializationZipperUtils;
private String typeAndIdSplit = "_";
public MaterializationConfServiceImpl(MaterializationRepository materializationRepository,
MaterializationElementRepository materializationElementRepository,
ModelService modelService, DatasourceService datasourceService) {
MaterializationElementRepository materializationElementRepository,
ModelService modelService, DatasourceService datasourceService,
MaterializationZipperUtils materializationZipperUtils) {
this.materializationRepository = materializationRepository;
this.materializationElementRepository = materializationElementRepository;
this.modelService = modelService;
this.datasourceService = datasourceService;
this.materializationZipperUtils = materializationZipperUtils;
}
@Override
@@ -145,7 +148,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
}
private void fillElementInfo(List<MaterializationElementResp> materializationElementRespList,
Map<String, SchemaItem> keyAndSchemaItemPair) {
Map<String, SchemaItem> keyAndSchemaItemPair) {
if (CollectionUtils.isEmpty(materializationElementRespList) || Objects.isNull(keyAndSchemaItemPair)) {
return;
}
@@ -243,8 +246,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
}
private String generateCreateSql(MaterializationResp materializationResp) {
MaterializationUtils materializationUtils = new MaterializationZipperUtils();
return materializationUtils.generateCreateSql(materializationResp);
return materializationZipperUtils.generateCreateSql(materializationResp);
}
@Override
@@ -260,7 +262,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
}
private void doDimensionMaterializationLogic(List<DimSchemaResp> dimensions,
MaterializationResp materializationResp, User user) {
MaterializationResp materializationResp, User user) {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}

37
semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java
View File

@@ -1,32 +1,37 @@
package com.tencent.supersonic.semantic.materialization.domain.utils;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementTypeEnum;
import com.tencent.supersonic.semantic.api.materialization.response.MaterializationElementResp;
import com.tencent.supersonic.semantic.api.materialization.response.MaterializationResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.stream.Collectors;
@Component
public class MaterializationZipperUtils implements MaterializationUtils {
private String split = "_";
private String createPatter = "CREATE TABLE IF NOT EXISTS `#{tableName}` (\n"
+ " `end_date` date NOT NULL COMMENT '日期',\n"
+ " `end_sys_imp_date` date NOT NULL COMMENT '日期',\n"
+ " `id` int(11) NOT NULL COMMENT 'id',\n"
+ " `start_date` date NULL,\n"
+ " `start_sys_imp_date` date NULL,\n"
+ " #{columnInfo}\n"
+ " ) ENGINE=OLAP\n"
+ "UNIQUE KEY(`end_date`, `id`)\n"
+ "UNIQUE KEY(`end_sys_imp_date`, `id`)\n"
+ "COMMENT 'OLAP'\n"
+ "PARTITION BY RANGE(`end_date`)\n"
+ "PARTITION BY RANGE(`end_sys_imp_date`)\n"
+ "(PARTITION p99991230 VALUES [('9999-12-30'), ('9999-12-31')))\n"
+ "DISTRIBUTED BY HASH(`id`) BUCKETS 9\n"
+ "PROPERTIES (\n"
@@ -40,6 +45,13 @@ public class MaterializationZipperUtils implements MaterializationUtils {
+ "\"enable_single_replica_compaction\" = \"false\"\n"
+ ");";
private final DimensionService dimensionService;
public MaterializationZipperUtils(DimensionService dimensionService) {
this.dimensionService = dimensionService;
}
@Override
public String generateCreateSql(MaterializationResp materializationResp) {
List<MaterializationElementResp> materializationElementRespList = materializationResp
@@ -48,6 +60,8 @@ public class MaterializationZipperUtils implements MaterializationUtils {
return "";
}
StringJoiner joiner = new StringJoiner(",");
Map<Long, DimensionResp> dimIdAndDim = dimensionService.getDimensions(materializationResp.getModelId())
.stream().collect(Collectors.toMap(DimensionResp::getId, value -> value, (v1, v2) -> v2));
materializationElementRespList.stream()
.filter(element -> TypeEnums.DIMENSION.equals(element.getType()) && ElementFrequencyEnum.LOW.equals(
element.getFrequency()))
@@ -57,11 +71,18 @@ public class MaterializationZipperUtils implements MaterializationUtils {
element.getElementType())) {
type = "date";
}
String description = element.getDescription().replace("'", "").replace("\"", "");
joiner.add(
String.format(" %s %s COMMENT '%s'",
element.getBizName(), type, description));
if (dimIdAndDim.containsKey(element.getId())) {
if (DataTypeEnums.ARRAY.equals(dimIdAndDim.get(element.getId()).getDataType())) {
type = "array<varchar(10000)>";
}
}
String description = element.getDescription().replace("'", "")
.replace("\"", "");
joiner.add(
String.format(" `%s` %s COMMENT '%s'",
element.getBizName(), type, description));
}
);
if (Strings.isEmpty(joiner.toString())) {