From e44e7ca8d5a47d6a93e3c9707137309784567bf9 Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Wed, 25 Oct 2023 20:21:24 +0800 Subject: [PATCH] [improvement](materialization) materializationZipperUtils support array type (#289) --- .../MaterializationConfServiceImpl.java | 18 +++++---- .../utils/MaterializationZipperUtils.java | 37 +++++++++++++++---- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java index 31ac51daa..4733c4930 100644 --- a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java +++ b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/application/MaterializationConfServiceImpl.java @@ -30,10 +30,10 @@ import com.tencent.supersonic.semantic.materialization.domain.pojo.Materializati import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationElementRepository; import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationRepository; import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationConverter; -import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationUtils; import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationZipperUtils; import com.tencent.supersonic.semantic.model.domain.DatasourceService; import com.tencent.supersonic.semantic.model.domain.ModelService; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -44,6 +44,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -58,16 +59,18 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic private final MaterializationElementRepository materializationElementRepository; private final ModelService modelService; private final DatasourceService datasourceService; + private final MaterializationZipperUtils materializationZipperUtils; private String typeAndIdSplit = "_"; public MaterializationConfServiceImpl(MaterializationRepository materializationRepository, - MaterializationElementRepository materializationElementRepository, - ModelService modelService, DatasourceService datasourceService) { + MaterializationElementRepository materializationElementRepository, + ModelService modelService, DatasourceService datasourceService, + MaterializationZipperUtils materializationZipperUtils) { this.materializationRepository = materializationRepository; this.materializationElementRepository = materializationElementRepository; - this.modelService = modelService; this.datasourceService = datasourceService; + this.materializationZipperUtils = materializationZipperUtils; } @Override @@ -145,7 +148,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic } private void fillElementInfo(List materializationElementRespList, - Map keyAndSchemaItemPair) { + Map keyAndSchemaItemPair) { if (CollectionUtils.isEmpty(materializationElementRespList) || Objects.isNull(keyAndSchemaItemPair)) { return; } @@ -243,8 +246,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic } private String generateCreateSql(MaterializationResp materializationResp) { - MaterializationUtils materializationUtils = new MaterializationZipperUtils(); - return materializationUtils.generateCreateSql(materializationResp); + return materializationZipperUtils.generateCreateSql(materializationResp); } @Override @@ -260,7 +262,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic } private void doDimensionMaterializationLogic(List dimensions, - MaterializationResp materializationResp, User user) { + MaterializationResp materializationResp, User user) { if (CollectionUtils.isEmpty(dimensions)) { return; } diff --git a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java index b27e290ce..d68ba2b97 100644 --- a/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java +++ b/semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java @@ -1,32 +1,37 @@ package com.tencent.supersonic.semantic.materialization.domain.utils; +import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum; import com.tencent.supersonic.semantic.api.materialization.enums.ElementTypeEnum; import com.tencent.supersonic.semantic.api.materialization.response.MaterializationElementResp; import com.tencent.supersonic.semantic.api.materialization.response.MaterializationResp; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.model.domain.DimensionService; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.StringJoiner; +import java.util.stream.Collectors; @Component public class MaterializationZipperUtils implements MaterializationUtils { private String split = "_"; private String createPatter = "CREATE TABLE IF NOT EXISTS `#{tableName}` (\n" - + " `end_date` date NOT NULL COMMENT '日期',\n" + + " `end_sys_imp_date` date NOT NULL COMMENT '日期',\n" + " `id` int(11) NOT NULL COMMENT 'id',\n" - + " `start_date` date NULL,\n" + + " `start_sys_imp_date` date NULL,\n" + " #{columnInfo}\n" + " ) ENGINE=OLAP\n" - + "UNIQUE KEY(`end_date`, `id`)\n" + + "UNIQUE KEY(`end_sys_imp_date`, `id`)\n" + "COMMENT 'OLAP'\n" - + "PARTITION BY RANGE(`end_date`)\n" + + "PARTITION BY RANGE(`end_sys_imp_date`)\n" + "(PARTITION p99991230 VALUES [('9999-12-30'), ('9999-12-31')))\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 9\n" + "PROPERTIES (\n" @@ -40,6 +45,13 @@ public class MaterializationZipperUtils implements MaterializationUtils { + "\"enable_single_replica_compaction\" = \"false\"\n" + ");"; + + private final DimensionService dimensionService; + + public MaterializationZipperUtils(DimensionService dimensionService) { + this.dimensionService = dimensionService; + } + @Override public String generateCreateSql(MaterializationResp materializationResp) { List materializationElementRespList = materializationResp @@ -48,6 +60,8 @@ public class MaterializationZipperUtils implements MaterializationUtils { return ""; } StringJoiner joiner = new StringJoiner(","); + Map dimIdAndDim = dimensionService.getDimensions(materializationResp.getModelId()) + .stream().collect(Collectors.toMap(DimensionResp::getId, value -> value, (v1, v2) -> v2)); materializationElementRespList.stream() .filter(element -> TypeEnums.DIMENSION.equals(element.getType()) && ElementFrequencyEnum.LOW.equals( element.getFrequency())) @@ -57,11 +71,18 @@ public class MaterializationZipperUtils implements MaterializationUtils { element.getElementType())) { type = "date"; } - String description = element.getDescription().replace("'", "").replace("\"", ""); - joiner.add( - String.format(" %s %s COMMENT '%s'", - element.getBizName(), type, description)); + if (dimIdAndDim.containsKey(element.getId())) { + if (DataTypeEnums.ARRAY.equals(dimIdAndDim.get(element.getId()).getDataType())) { + type = "array"; + } + } + String description = element.getDescription().replace("'", "") + .replace("\"", ""); + joiner.add( + String.format(" `%s` %s COMMENT '%s'", + element.getBizName(), type, description)); + } ); if (Strings.isEmpty(joiner.toString())) {