From 276009f014a0f8fd485878fd69a703c361d861f4 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:06:35 +0800 Subject: [PATCH] [improvement](Headless) add tags to ModelDemoDataLoader (#795) --- .../service/impl/SchemaServiceImpl.java | 4 +- .../supersonic/BenchMarkDemoDataLoader.java | 4 +- .../supersonic/DuSQLDemoDataLoader.java | 4 +- .../supersonic/HeadlessDemoLoader.java | 4 - .../supersonic/ModelDemoDataLoader.java | 94 ++++++++++++++----- 5 files changed, 77 insertions(+), 33 deletions(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index a48672fb0..d906134c0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -159,8 +159,8 @@ public class SchemaServiceImpl implements SchemaService { dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); - tagRespList = TagConverter.filterByDataSet(tagRespList, dataSetResp); - dataSetSchemaResp.setTags(tagRespList); + List tagResps = TagConverter.filterByDataSet(tagRespList, dataSetResp); + dataSetSchemaResp.setTags(tagResps); dataSetSchemaResps.add(dataSetSchemaResp); } fillStaticInfo(dataSetSchemaResps); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index 1eaead5b2..51d492f50 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -61,7 +61,7 @@ public class BenchMarkDemoDataLoader { addModel_2(); addModel_3(); addModel_4(); - addView_1(); + addDataSet_1(); addModelRela_1(); addModelRela_2(); addModelRela_3(); @@ -213,7 +213,7 @@ public class BenchMarkDemoDataLoader { modelService.createModel(modelReq, user); } - public void addView_1() { + public void addDataSet_1() { DataSetReq viewReq = new DataSetReq(); viewReq.setName("cspider"); viewReq.setBizName("singer"); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java index e8da2098f..f9fb33387 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java @@ -65,7 +65,7 @@ public class DuSQLDemoDataLoader { addModel_2(); addModel_3(); addModel_4(); - addView_1(); + addDataSet_1(); addModelRela_1(); addModelRela_2(); addModelRela_3(); @@ -254,7 +254,7 @@ public class DuSQLDemoDataLoader { } - public void addView_1() { + public void addDataSet_1() { DataSetReq viewReq = new DataSetReq(); viewReq.setName("DuSQL 互联网企业"); viewReq.setBizName("internet"); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/HeadlessDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/HeadlessDemoLoader.java index bc6b64a25..1cde40f9c 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/HeadlessDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/HeadlessDemoLoader.java @@ -25,9 +25,6 @@ public class HeadlessDemoLoader implements CommandLineRunner { @Autowired private BenchMarkDemoDataLoader benchMarkDemoLoader; - @Autowired - private DuSQLDemoDataLoader duSQLDemoDataLoader; - @Value("${demo.enabled:false}") private boolean demoEnabled; @@ -39,7 +36,6 @@ public class HeadlessDemoLoader implements CommandLineRunner { } modelDataDemoLoader.doRun(); benchMarkDemoLoader.doRun(); - //duSQLDemoDataLoader.doRun(); isLoad = true; } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index cca577fbe..45269cf16 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -14,16 +14,9 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.DefaultDisplayInfo; -import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.QueryConfig; -import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.enums.DataType; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; @@ -36,34 +29,44 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.MetricParam; +import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.RelateDimension; -import com.tencent.supersonic.headless.api.pojo.DataSetDetail; -import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; +import com.tencent.supersonic.headless.api.pojo.TagDefineParams; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.enums.DataType; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; +import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; +import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.DomainReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.request.TagReq; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.DataSetService; +import com.tencent.supersonic.headless.server.service.TagMetaService; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - @Component @Slf4j public class ModelDemoDataLoader { @@ -88,6 +91,9 @@ public class ModelDemoDataLoader { @Autowired private DataSourceProperties dataSourceProperties; + @Autowired + private TagMetaService tagMetaService; + public void doRun() { try { addDatabase(); @@ -104,8 +110,9 @@ public class ModelDemoDataLoader { updateDimension(); updateMetric(); updateMetric_pv(); - addView_1(); - addView_2(); + addTags(); + addDataSet_1(); + addDataSet_2(); addAuthGroup_1(); addAuthGroup_2(); } catch (Exception e) { @@ -399,6 +406,47 @@ public class ModelDemoDataLoader { metricService.updateMetric(metricReq, user); } + private void addTags() { + TagReq tagReq = new TagReq(); + tagReq.setModelId(4L); + tagReq.setName("活跃区域"); + tagReq.setBizName("act_area"); + tagReq.setStatus(StatusEnum.ONLINE.getCode()); + tagReq.setTypeEnum(TypeEnums.TAG); + tagReq.setTagDefineType(TagDefineType.DIMENSION); + TagDefineParams tagDefineParams = new TagDefineParams(); + tagDefineParams.setExpr("act_area"); + tagDefineParams.setDependencies(new ArrayList<>(Arrays.asList(4))); + tagReq.setTagDefineParams(tagDefineParams); + tagMetaService.create(tagReq, user); + + TagReq tagReq2 = new TagReq(); + tagReq2.setModelId(4L); + tagReq2.setName("风格"); + tagReq2.setBizName("genre"); + tagReq2.setStatus(StatusEnum.ONLINE.getCode()); + tagReq2.setTypeEnum(TypeEnums.TAG); + tagReq2.setTagDefineType(TagDefineType.DIMENSION); + TagDefineParams tagDefineParam2s = new TagDefineParams(); + tagDefineParam2s.setExpr("genre"); + tagDefineParam2s.setDependencies(new ArrayList<>(Arrays.asList(6))); + tagReq2.setTagDefineParams(tagDefineParam2s); + tagMetaService.create(tagReq2, user); + + TagReq tagReq3 = new TagReq(); + tagReq3.setModelId(4L); + tagReq3.setName("播放量"); + tagReq3.setBizName("js_play_cnt"); + tagReq3.setStatus(StatusEnum.ONLINE.getCode()); + tagReq3.setTypeEnum(TypeEnums.TAG); + tagReq3.setTagDefineType(TagDefineType.METRIC); + TagDefineParams tagDefineParam3s = new TagDefineParams(); + tagDefineParam3s.setExpr("js_play_cnt"); + tagDefineParam3s.setDependencies(new ArrayList<>(Arrays.asList(5))); + tagReq3.setTagDefineParams(tagDefineParam3s); + tagMetaService.create(tagReq3, user); + } + public void addMetric_uv() throws Exception { MetricReq metricReq = new MetricReq(); metricReq.setModelId(2L); @@ -445,7 +493,7 @@ public class ModelDemoDataLoader { metricService.createMetric(metricReq, user); } - public void addView_1() { + public void addDataSet_1() { DataSetReq dataSetReq = new DataSetReq(); dataSetReq.setName("超音数"); dataSetReq.setBizName("s2"); @@ -472,7 +520,7 @@ public class ModelDemoDataLoader { dataSetService.save(dataSetReq, User.getFakeUser()); } - public void addView_2() { + public void addDataSet_2() { DataSetReq dataSetReq = new DataSetReq(); dataSetReq.setName("艺人库"); dataSetReq.setBizName("singer"); @@ -481,7 +529,7 @@ public class ModelDemoDataLoader { dataSetReq.setAdmins(Lists.newArrayList("admin", "jack")); List dataSetModelConfigs = Lists.newArrayList( new DataSetModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), - Lists.newArrayList(5L, 6L, 7L), Lists.newArrayList()) + Lists.newArrayList(5L, 6L, 7L), Lists.newArrayList(1L, 2L, 3L)) ); DataSetDetail dataSetDetail = new DataSetDetail(); dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs);