From fc82350af5c6cb224539953f66e5c2d7e0da8454 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 19 Jan 2024 17:39:37 +0800 Subject: [PATCH] (improvement)(headless) Update data demo (#659) Co-authored-by: jolunoluo --- .../headless/api/pojo/FieldParam.java | 2 + .../headless/api/pojo/MetricParam.java | 4 + .../service/impl/DimensionServiceImpl.java | 4 +- .../headless/server/utils/ModelConverter.java | 28 ++++++- .../supersonic/ModelDemoDataLoader.java | 75 +++++++++++++++++-- 5 files changed, 99 insertions(+), 14 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java index 4e3b6bf0e..054969b82 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/FieldParam.java @@ -1,8 +1,10 @@ package com.tencent.supersonic.headless.api.pojo; +import lombok.Builder; import lombok.Data; @Data +@Builder public class FieldParam { private String fieldName; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java index 2e2b696bb..8d635cf48 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricParam.java @@ -1,8 +1,12 @@ package com.tencent.supersonic.headless.api.pojo; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor public class MetricParam { private Long id; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 4d4002395..4a1f6aa1c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -339,14 +339,14 @@ public class DimensionServiceImpl implements DimensionService { if (bizNameMap.containsKey(dimensionReq.getBizName())) { DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName()); if (!dimensionResp.getId().equals(dimensionReq.getId())) { - throw new RuntimeException(String.format("该模型下存在相同的维度字段名:%s 创建人:%s", + throw new RuntimeException(String.format("该主题域下存在相同的维度字段名:%s 创建人:%s", dimensionReq.getBizName(), dimensionResp.getCreatedBy())); } } if (nameMap.containsKey(dimensionReq.getName())) { DimensionResp dimensionResp = nameMap.get(dimensionReq.getName()); if (!dimensionResp.getId().equals(dimensionReq.getId())) { - throw new RuntimeException(String.format("该模型下存在相同的维度名:%s 创建人:%s", + throw new RuntimeException(String.format("该主题域下存在相同的维度名:%s 创建人:%s", dimensionReq.getName(), dimensionResp.getCreatedBy())); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index 925ce459e..c27ed2ee6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -40,7 +40,7 @@ public class ModelConverter { public static ModelDO convert(ModelReq modelReq, User user) { ModelDO modelDO = new ModelDO(); - ModelDetail modelDetail = getModelDetail(modelReq); + ModelDetail modelDetail = createModelDetail(modelReq); modelReq.createdBy(user.getName()); BeanMapper.mapper(modelReq, modelDO); modelDO.setStatus(StatusEnum.ONLINE.getCode()); @@ -76,7 +76,7 @@ public class ModelConverter { } public static ModelDO convert(ModelDO modelDO, ModelReq modelReq, User user) { - ModelDetail modelDetail = getModelDetail(modelReq); + ModelDetail modelDetail = updateModelDetail(modelReq); BeanMapper.mapper(modelReq, modelDO); if (modelReq.getDrillDownDimensions() != null) { modelDO.setDrillDownDimensions(JSONObject.toJSONString(modelReq.getDrillDownDimensions())); @@ -114,7 +114,7 @@ public class ModelConverter { public static MetricReq convert(Measure measure, ModelDO modelDO) { MetricReq metricReq = new MetricReq(); metricReq.setName(measure.getName()); - metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", "")); + metricReq.setBizName(measure.getExpr()); metricReq.setDescription(measure.getName()); metricReq.setModelId(modelDO.getId()); MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams(); @@ -200,9 +200,29 @@ public class ModelConverter { return measures.stream().map(measure -> convert(measure, modelDO)).collect(Collectors.toList()); } - private static ModelDetail getModelDetail(ModelReq modelReq) { + private static ModelDetail createModelDetail(ModelReq modelReq) { ModelDetail modelDetail = new ModelDetail(); List measures = modelReq.getModelDetail().getMeasures(); + if (measures == null) { + measures = Lists.newArrayList(); + } + for (Measure measure : measures) { + if (StringUtils.isBlank(measure.getBizName())) { + continue; + } + measure.setExpr(measure.getBizName()); + measure.setBizName(String.format("%s_%s", modelReq.getBizName(), measure.getExpr())); + } + BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); + return modelDetail; + } + + private static ModelDetail updateModelDetail(ModelReq modelReq) { + ModelDetail modelDetail = new ModelDetail(); + List measures = modelReq.getModelDetail().getMeasures(); + if (measures == null) { + measures = Lists.newArrayList(); + } for (Measure measure : measures) { if (StringUtils.isBlank(measure.getBizName())) { continue; diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index bd796d71c..082aa58a8 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -19,10 +19,15 @@ import com.tencent.supersonic.headless.api.enums.MetricDefineType; import com.tencent.supersonic.headless.api.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; +import com.tencent.supersonic.headless.api.pojo.Field; +import com.tencent.supersonic.headless.api.pojo.FieldParam; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MeasureParam; +import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; +import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; +import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.request.DimensionReq; @@ -74,6 +79,8 @@ public class ModelDemoDataLoader { addDomain(); addModel_1(); addModel_2(); + addMetric_uv(); + addMetric_pv_avg(); addModel_3(); addModelRela_1(); addModelRela_2(); @@ -140,7 +147,10 @@ public class ModelDemoDataLoader { dimensions.add(new Dim("部门", "department", DimensionType.categorical.name(), 1)); modelDetail.setDimensions(dimensions); - + List fields = Lists.newArrayList(); + fields.add(Field.builder().fieldName("user_name").dataType("Varchar").build()); + fields.add(Field.builder().fieldName("department").dataType("Varchar").build()); + modelDetail.setFields(fields); modelDetail.setMeasures(Collections.emptyList()); modelDetail.setQueryType("sql_query"); modelDetail.setSqlQuery("select user_name,department from s2_user_department"); @@ -172,17 +182,21 @@ public class ModelDemoDataLoader { dimension2.setExpr("page"); dimensions.add(dimension2); modelDetail.setDimensions(dimensions); - List measures = new ArrayList<>(); Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1); measures.add(measure1); - - Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1); + Measure measure2 = new Measure("访问用户数", "user_id", AggOperatorEnum.SUM.name(), 0); measures.add(measure2); - modelDetail.setMeasures(measures); + List fields = Lists.newArrayList(); + fields.add(Field.builder().fieldName("s2_pv_uv_statis_user_name").dataType("Varchar").build()); + fields.add(Field.builder().fieldName("imp_date").dataType("Date").build()); + fields.add(Field.builder().fieldName("page").dataType("Varchar").build()); + fields.add(Field.builder().fieldName("pv").dataType("Long").build()); + fields.add(Field.builder().fieldName("user_id").dataType("Varchar").build()); + modelDetail.setFields(fields); modelDetail.setSqlQuery("SELECT imp_date, user_name as s2_pv_uv_statis_user_name, page, 1 as pv, " - + "user_name as uv FROM s2_pv_uv_statis"); + + "user_name as user_id FROM s2_pv_uv_statis"); modelDetail.setQueryType("sql_query"); modelReq.setDomainId(1L); modelReq.setModelDetail(modelDetail); @@ -216,8 +230,13 @@ public class ModelDemoDataLoader { List measures = new ArrayList<>(); Measure measure1 = new Measure("停留时长", "stay_hours", AggregateTypeEnum.SUM.name(), 1); measures.add(measure1); - modelDetail.setMeasures(measures); + List fields = Lists.newArrayList(); + fields.add(Field.builder().fieldName("stay_hours_user_name").dataType("Varchar").build()); + fields.add(Field.builder().fieldName("imp_date").dataType("Date").build()); + fields.add(Field.builder().fieldName("page").dataType("Varchar").build()); + fields.add(Field.builder().fieldName("stay_hours").dataType("Double").build()); + modelDetail.setFields(fields); modelDetail.setSqlQuery( "select imp_date,user_name as stay_hours_user_name,stay_hours,page from s2_stay_time_statis"); modelDetail.setQueryType("sql_query"); @@ -324,7 +343,7 @@ public class ModelDemoDataLoader { public void updateMetric() throws Exception { MetricReq metricReq = new MetricReq(); metricReq.setModelId(3L); - metricReq.setId(3L); + metricReq.setId(4L); metricReq.setName("停留时长"); metricReq.setBizName("stay_hours"); metricReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); @@ -343,6 +362,46 @@ public class ModelDemoDataLoader { metricService.updateMetric(metricReq, user); } + public void addMetric_uv() throws Exception { + MetricReq metricReq = new MetricReq(); + metricReq.setModelId(2L); + metricReq.setName("访问用户数"); + metricReq.setBizName("uv"); + metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode()); + metricReq.setDescription("访问的用户个数"); + metricReq.setAlias("UV"); + MetricDefineByFieldParams metricTypeParams = new MetricDefineByFieldParams(); + metricTypeParams.setExpr("count(distinct user_id)"); + List fieldParams = new ArrayList<>(); + fieldParams.add(FieldParam.builder().fieldName("user_id").build()); + metricTypeParams.setFields(fieldParams); + metricReq.setMetricDefineByFieldParams(metricTypeParams); + metricReq.setMetricDefineType(MetricDefineType.FIELD); + metricService.createMetric(metricReq, user); + } + + public void addMetric_pv_avg() throws Exception { + MetricReq metricReq = new MetricReq(); + metricReq.setModelId(2L); + metricReq.setName("人均访问次数"); + metricReq.setBizName("pv_avg"); + metricReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode()); + metricReq.setDescription("每个用户平均访问的次数"); + metricReq.setTags(Collections.singletonList("核心指标")); + metricReq.setAlias("平均访问次数"); + MetricDefineByMetricParams metricTypeParams = new MetricDefineByMetricParams(); + metricTypeParams.setExpr("pv/uv"); + List metrics = new ArrayList<>(); + MetricParam metricPv = new MetricParam(1L, "pv"); + MetricParam metricUv = new MetricParam(2L, "uv"); + metrics.add(metricPv); + metrics.add(metricUv); + metricTypeParams.setMetrics(metrics); + metricReq.setMetricDefineByMetricParams(metricTypeParams); + metricReq.setMetricDefineType(MetricDefineType.METRIC); + metricService.createMetric(metricReq, user); + } + public void addAuthGroup_1() { AuthGroup authGroupReq = new AuthGroup(); authGroupReq.setModelId(3L);