From b40670b0e3e1bfd8be64353e675a826de42c2f29 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 19 Jan 2024 14:51:13 +0800 Subject: [PATCH] (improvement)(headless) (improvement)(headless) Supports creating new metric by fields and metrics and convert struct to sql (#654) Co-authored-by: jolunoluo --- .../supersonic/headless/api/pojo/Field.java | 4 ++ .../supersonic/headless/api/pojo/Measure.java | 2 - .../headless/api/pojo/MetricDefineParams.java | 2 + .../headless/api/pojo/ModelDetail.java | 24 ++++++++ .../headless/api/request/FieldRemovedReq.java | 13 ++++ .../headless/api/request/MetricReq.java | 13 ++-- .../headless/api/request/QueryStructReq.java | 16 +++-- .../headless/api/response/MetricResp.java | 28 ++++++--- .../api/response/UnAvailableItemResp.java | 16 +++++ .../core/manager/MetricYamlManager.java | 2 +- .../pojo/yaml/MetricTypeParamsYamlTpl.java | 7 ++- .../headless/core/utils/SqlGenerateUtils.java | 35 +++++------ .../server/manager/SemanticSchemaManager.java | 22 ++++--- .../headless/server/pojo/MetaFilter.java | 2 + .../server/rest/MetricController.java | 4 +- .../headless/server/rest/ModelController.java | 7 +++ .../headless/server/service/ModelService.java | 5 +- .../service/impl/DimensionServiceImpl.java | 18 +++++- .../service/impl/MetricServiceImpl.java | 36 ++++++++++- .../server/service/impl/ModelServiceImpl.java | 16 ++--- .../server/service/impl/QueryServiceImpl.java | 61 ++++++------------- .../server/utils/MetricCheckUtils.java | 2 +- .../server/utils/MetricConverter.java | 2 +- .../headless/server/utils/ModelConverter.java | 6 +- .../server/utils/QueryReqConverter.java | 5 +- .../server/service/MetricServiceImplTest.java | 6 +- .../supersonic/ModelDemoDataLoader.java | 2 +- 27 files changed, 238 insertions(+), 118 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java index 6e289f4e8..6ecb48151 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java @@ -1,10 +1,14 @@ package com.tencent.supersonic.headless.api.pojo; +import lombok.Builder; import lombok.Data; @Data +@Builder public class Field { private String fieldName; + private String dataType; + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java index d61299c2b..e71f23c60 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java @@ -15,8 +15,6 @@ public class Measure { private String expr; - private String createMetric; - private String bizName; private Integer isCreateMetric = 0; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java index 257f73d0e..47783f5dc 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java @@ -7,4 +7,6 @@ public abstract class MetricDefineParams { private String expr; + private String filterSql; + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 3d1bc5094..9f1732e0c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -43,4 +43,28 @@ public class ModelDetail { .collect(Collectors.toList()); } + public List getFields() { + if (!CollectionUtils.isEmpty(fields)) { + return fields; + } + List fieldList = Lists.newArrayList(); + //Compatible with older versions + if (!CollectionUtils.isEmpty(identifiers)) { + fieldList.addAll(identifiers.stream() + .map(identify -> Field.builder().fieldName(identify.getFieldName()).build()) + .collect(Collectors.toSet())); + } + if (!CollectionUtils.isEmpty(dimensions)) { + fieldList.addAll(dimensions.stream() + .map(dim -> Field.builder().fieldName(dim.getFieldName()).build()) + .collect(Collectors.toSet())); + } + if (!CollectionUtils.isEmpty(measures)) { + fieldList.addAll(measures.stream() + .map(measure -> Field.builder().fieldName(measure.getFieldName()).build()) + .collect(Collectors.toSet())); + } + return fieldList; + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java new file mode 100644 index 000000000..1fb38e764 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.headless.api.request; + +import lombok.Data; +import java.util.List; + +@Data +public class FieldRemovedReq { + + private Long modelId; + + private List fields; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java index 834bee10d..6e70dc183 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java @@ -12,23 +12,24 @@ import lombok.Data; public class MetricReq extends MetricBaseReq { private MetricDefineType metricDefineType = MetricDefineType.MEASURE; - private MetricDefineByMeasureParams typeParams; + private MetricDefineByMeasureParams metricDefineByMeasureParams; private MetricDefineByFieldParams metricDefineByFieldParams; private MetricDefineByMetricParams metricDefineByMetricParams; public String getTypeParamsJson() { - if (metricDefineByFieldParams != null) { + if (MetricDefineType.FIELD.equals(metricDefineType) && metricDefineByFieldParams != null) { return JSONObject.toJSONString(metricDefineByFieldParams); - } else if (typeParams != null) { - return JSONObject.toJSONString(typeParams); - } else if (metricDefineByMetricParams != null) { + } else if (MetricDefineType.MEASURE.equals(metricDefineType) && metricDefineByMeasureParams != null) { + return JSONObject.toJSONString(metricDefineByMeasureParams); + } else if (MetricDefineType.METRIC.equals(metricDefineType) && metricDefineByMetricParams != null) { return JSONObject.toJSONString(metricDefineByMetricParams); } return null; } public MetricType getMetricType() { - return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC; + return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams) + ? MetricType.DERIVED : MetricType.ATOMIC; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java index bf971de12..4765c44d8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java @@ -12,10 +12,6 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -39,6 +35,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + @Data @Slf4j @@ -266,4 +267,11 @@ public class QueryStructReq extends SemanticQueryReq { return sql; } + public String getModelName() { + if (StringUtils.isNotBlank(modelName)) { + return modelName; + } + return "table"; + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java index 4b50abd59..8378a56a9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java @@ -10,15 +10,16 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import lombok.Data; +import lombok.ToString; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.Data; -import lombok.ToString; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; @Data @@ -52,7 +53,7 @@ public class MetricResp extends SchemaItem { private MetricDefineType metricDefineType = MetricDefineType.MEASURE; - private MetricDefineByMeasureParams typeParams; + private MetricDefineByMeasureParams metricDefineByMeasureParams; private MetricDefineByFieldParams metricDefineByFieldParams; @@ -77,9 +78,20 @@ public class MetricResp extends SchemaItem { } public String getDefaultAgg() { - if (typeParams != null - && CollectionUtils.isNotEmpty(typeParams.getMeasures())) { - return typeParams.getMeasures().get(0).getAgg(); + if (metricDefineByMeasureParams != null + && CollectionUtils.isNotEmpty(metricDefineByMeasureParams.getMeasures())) { + return metricDefineByMeasureParams.getMeasures().get(0).getAgg(); + } + return ""; + } + + public String getExpr() { + if (MetricDefineType.MEASURE.equals(metricDefineType)) { + return metricDefineByMeasureParams.getExpr(); + } else if (MetricDefineType.METRIC.equals(metricDefineType)) { + return metricDefineByMetricParams.getExpr(); + } else if (MetricDefineType.FIELD.equals(metricDefineType)) { + return metricDefineByFieldParams.getExpr(); } return ""; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java new file mode 100644 index 000000000..f7d1da66a --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.api.response; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@Builder +public class UnAvailableItemResp { + + private List metricResps; + + private List dimensionResps; + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java index b5c5bb76e..f9e4d1b60 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java @@ -46,7 +46,7 @@ public class MetricYamlManager { metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) { - MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams(); + MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams(); metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); List measures = metricDefineParams.getMeasures(); metricTypeParamsYamlTpl.setMeasures( diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java index 832947c99..5fc59ca81 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java @@ -1,17 +1,18 @@ package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.Data; +import org.apache.commons.compress.utils.Lists; import java.util.List; @Data public class MetricTypeParamsYamlTpl { - private List measures; + private List measures = Lists.newArrayList(); - private List metrics; + private List metrics = Lists.newArrayList(); - private List fields; + private List fields = Lists.newArrayList(); private String expr; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 264931a61..a0153cb3d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -1,12 +1,5 @@ package com.tencent.supersonic.headless.core.utils; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; @@ -23,6 +16,15 @@ import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Collections; @@ -34,14 +36,13 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Triple; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; /** * tools functions to analyze queryStructReq @@ -333,7 +334,7 @@ public class SqlGenerateUtils { public String getExpr(MetricResp metricResp) { if (Objects.isNull(metricResp.getMetricDefineType())) { - return metricResp.getTypeParams().getExpr(); + return metricResp.getMetricDefineByMeasureParams().getExpr(); } if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) { return metricResp.getMetricDefineByMetricParams().getExpr(); @@ -342,6 +343,6 @@ public class SqlGenerateUtils { return metricResp.getMetricDefineByFieldParams().getExpr(); } // measure add agg function - return metricResp.getTypeParams().getExpr(); + return metricResp.getMetricDefineByMeasureParams().getExpr(); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 41ae1d57e..27df67414 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -31,6 +31,16 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -43,15 +53,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Slf4j @@ -142,7 +143,8 @@ public class SemanticSchemaManager { || identifiers.contains(f.getFieldName())) { continue; } - datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build()); + datasource.getMeasures().add(Measure.builder().name(f.getFieldName()) + .expr(f.getFieldName()).agg("").build()); } } return datasource; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java index 4971c218f..3b507f235 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java @@ -27,6 +27,8 @@ public class MetaFilter { private List ids; + private List fieldsDepend; + public MetaFilter(List modelIds) { this.modelIds = modelIds; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java index ced5eb206..39f6e0d4c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java @@ -40,7 +40,7 @@ public class MetricController { this.metricService = metricService; } - @PostMapping("/creatExprMetric") + @PostMapping("/createMetric") public MetricResp createMetric(@RequestBody MetricReq metricReq, HttpServletRequest request, HttpServletResponse response) throws Exception { @@ -48,7 +48,7 @@ public class MetricController { return metricService.createMetric(metricReq, user); } - @PostMapping("/updateExprMetric") + @PostMapping("/updateMetric") public MetricResp updateMetric(@RequestBody MetricReq metricReq, HttpServletRequest request, HttpServletResponse response) throws Exception { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java index 5d52ab33b..4d7fa25d3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java @@ -3,10 +3,12 @@ package com.tencent.supersonic.headless.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.headless.api.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.ModelReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.ModelResp; +import com.tencent.supersonic.headless.api.response.UnAvailableItemResp; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.ModelService; import org.springframework.web.bind.annotation.DeleteMapping; @@ -92,4 +94,9 @@ public class ModelController { return true; } + @PostMapping("/getUnAvailableItem") + public UnAvailableItemResp getUnAvailableItem(@RequestBody FieldRemovedReq fieldRemovedReq) { + return modelService.getUnAvailableItem(fieldRemovedReq); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index 24e804490..9c8062208 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -4,13 +4,14 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.ModelReq; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.headless.api.response.MeasureResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.response.UnAvailableItemResp; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; @@ -34,7 +35,7 @@ public interface ModelService { ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); - List getMeasureListOfModel(List modelIds); + UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq); List getModelListWithAuth(User user, Long domainId, AuthType authType); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 60f236864..4d4002395 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -215,13 +215,29 @@ public class DimensionServiceImpl implements DimensionService { DimensionFilter dimensionFilter = new DimensionFilter(); BeanUtils.copyProperties(metaFilter, dimensionFilter); List dimensionDOS = dimensionRepository.getDimension(dimensionFilter); - return convertList(dimensionDOS, modelService.getModelMap()); + List dimensionResps = convertList(dimensionDOS, modelService.getModelMap()); + if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { + return filterByField(dimensionResps, metaFilter.getFieldsDepend()); + } + return dimensionResps; } private List getDimensions(Long modelId) { return getDimensions(new MetaFilter(Lists.newArrayList(modelId))); } + private List filterByField(List dimensionResps, List fields) { + List dimensionFiltered = Lists.newArrayList(); + for (DimensionResp dimensionResp : dimensionResps) { + for (String field : fields) { + if (dimensionResp.getExpr().contains(field)) { + dimensionFiltered.add(dimensionResp); + } + } + } + return dimensionFiltered; + } + @Override public List getDimensionInModelCluster(Long modelId) { ModelResp modelResp = modelService.getModel(modelId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 27b1fed2b..1dda69657 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -16,6 +16,7 @@ import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.headless.api.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.MetricBaseReq; @@ -206,7 +207,36 @@ public class MetricServiceImpl implements MetricService { public List getMetrics(MetaFilter metaFilter) { MetricFilter metricFilter = new MetricFilter(); BeanUtils.copyProperties(metaFilter, metricFilter); - return convertList(queryMetric(metricFilter), Lists.newArrayList()); + List metricResps = convertList(queryMetric(metricFilter)); + if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) { + return filterByField(metricResps, metaFilter.getFieldsDepend()); + } + return metricResps; + } + + private List filterByField(List metricResps, List fields) { + List metricRespFiltered = Lists.newArrayList(); + for (MetricResp metricResp : metricResps) { + for (String field : fields) { + if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { + List ids = metricResp.getMetricDefineByMetricParams().getMetrics() + .stream().map(MetricParam::getId).collect(Collectors.toList()); + List metricById = metricResps.stream() + .filter(metric -> ids.contains(metric.getId())) + .collect(Collectors.toList()); + for (MetricResp metric : metricById) { + if (metric.getExpr().contains(field)) { + metricRespFiltered.add(metricResp); + } + } + } else { + if (metricResp.getExpr().contains(field)) { + metricRespFiltered.add(metricResp); + } + } + } + } + return metricRespFiltered; } @Override @@ -362,6 +392,10 @@ public class MetricServiceImpl implements MetricService { return getMetrics(new MetaFilter(modelIds)); } + private List convertList(List metricDOS) { + return convertList(metricDOS, Lists.newArrayList()); + } + private List convertList(List metricDOS, List collect) { List metricResps = Lists.newArrayList(); Map modelMap = modelService.getModelMap(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index bf9a9677f..f3bb0046f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -17,6 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.request.DateInfoReq; import com.tencent.supersonic.headless.api.request.DimensionReq; +import com.tencent.supersonic.headless.api.request.FieldRemovedReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.ModelReq; @@ -25,11 +26,11 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; -import com.tencent.supersonic.headless.api.response.MeasureResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; +import com.tencent.supersonic.headless.api.response.UnAvailableItemResp; import com.tencent.supersonic.headless.core.manager.DimensionYamlManager; import com.tencent.supersonic.headless.core.manager.MetricYamlManager; import com.tencent.supersonic.headless.core.manager.ModelYamlManager; @@ -197,12 +198,13 @@ public class ModelServiceImpl implements ModelService { } @Override - public List getMeasureListOfModel(List modelIds) { - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setIds(modelIds); - List modelResps = getModelList(modelFilter); - return modelResps.stream().flatMap(modelResp -> modelResp.getModelDetail().getMeasures() - .stream().map(measure -> ModelConverter.convert(measure, modelResp))).collect(Collectors.toList()); + public UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq) { + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(fieldRemovedReq.getModelId())); + metaFilter.setFieldsDepend(fieldRemovedReq.getFields()); + List metricResps = metricService.getMetrics(metaFilter); + List dimensionResps = dimensionService.getDimensions(metaFilter); + return UnAvailableItemResp.builder().dimensionResps(dimensionResps) + .metricResps(metricResps).build(); } private void batchCreateDimension(ModelDO modelDO, User user) throws Exception { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index c5d5a65c5..1bed0bf5f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -155,49 +155,7 @@ public class QueryServiceImpl implements QueryService { @Override public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception { - SemanticQueryResp semanticQueryResp = null; - TaskStatusEnum state = TaskStatusEnum.SUCCESS; - log.info("[queryStructReq:{}]", queryStructReq); - try { - //1.initStatInfo - statUtils.initStatInfo(queryStructReq, user); - //2.query from cache - Object query = queryCache.query(queryStructReq); - if (Objects.nonNull(query)) { - return (SemanticQueryResp) query; - } - StatUtils.get().setUseResultCache(false); - //3 parse - QueryStatement queryStatement = buildQueryStatement(queryStructReq); - queryStatement = queryParser.parse(queryStatement); - - //4 plan - QueryExecutor queryExecutor = queryPlanner.plan(queryStatement); - - //5 execute - if (queryExecutor != null) { - semanticQueryResp = queryExecutor.execute(queryStatement); - if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { - queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds()); - } - } - //6 reset cache and set stateInfo - Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp); - if (setCacheSuccess) { - // if semanticQueryResp is not null, update cache data - statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq)); - } - if (Objects.isNull(semanticQueryResp)) { - state = TaskStatusEnum.ERROR; - } - return semanticQueryResp; - } catch (Exception e) { - log.error("exception in queryByStruct, e: ", e); - state = TaskStatusEnum.ERROR; - throw e; - } finally { - statUtils.statInfo2DbAsync(state); - } + return (SemanticQueryResp) queryBySql(queryStructReq.convert(queryStructReq), user); } private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception { @@ -419,6 +377,17 @@ public class QueryServiceImpl implements QueryService { return false; } + private SemanticQueryResp queryByCache(String key, Object queryCmd) { + + Object resultObject = cacheManager.get(key); + if (Objects.nonNull(resultObject)) { + log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString()); + statUtils.updateResultCacheKey(key); + return (SemanticQueryResp) resultObject; + } + return null; + } + private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { QuerySqlReq querySQLReq = new QuerySqlReq(); List modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); @@ -436,4 +405,10 @@ public class QueryServiceImpl implements QueryService { querySQLReq.setSql(sql); return querySQLReq; } + + private String getKeyByModelIds(List modelIds) { + return String.join(",", modelIds.stream() + .map(Object::toString).collect(Collectors.toList())); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java index 41fefd551..199a5e24d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java @@ -28,7 +28,7 @@ public class MetricCheckUtils { } } if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) { - MetricDefineByMeasureParams typeParams = metricReq.getTypeParams(); + MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams(); if (typeParams == null) { throw new InvalidArgumentException("指标定义参数不可为空"); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 2b5bed642..0dddd5209 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -81,7 +81,7 @@ public class MetricConverter { } metricResp.setTypeEnum(TypeEnums.METRIC); if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) { - metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), + metricResp.setMetricDefineByMeasureParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricDefineByMeasureParams.class)); } else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) { metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(), diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index 9b0e78ed2..925ce459e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -122,7 +122,7 @@ public class ModelConverter { MeasureParam measureParam = new MeasureParam(); BeanMapper.mapper(measure, measureParam); exprTypeParams.setMeasures(Lists.newArrayList(measureParam)); - metricReq.setTypeParams(exprTypeParams); + metricReq.setMetricDefineByMeasureParams(exprTypeParams); metricReq.setMetricDefineType(MetricDefineType.MEASURE); return metricReq; } @@ -202,8 +202,7 @@ public class ModelConverter { private static ModelDetail getModelDetail(ModelReq modelReq) { ModelDetail modelDetail = new ModelDetail(); - BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); - List measures = modelDetail.getMeasures(); + List measures = modelReq.getModelDetail().getMeasures(); for (Measure measure : measures) { if (StringUtils.isBlank(measure.getBizName())) { continue; @@ -216,6 +215,7 @@ public class ModelConverter { measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName)); } } + BeanMapper.mapper(modelReq.getModelDetail(), modelDetail); return modelDetail; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 15cc8805b..5304e8966 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -279,7 +279,7 @@ public class QueryReqConverter { // check metrics has derived if (!metricResps.stream() .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(), - m.getTypeParams()))) { + m.getMetricDefineByMeasureParams()))) { return; } Set allFields = new HashSet<>(); @@ -298,7 +298,8 @@ public class QueryReqConverter { if (!CollectionUtils.isEmpty(metricResps)) { for (MetricResp metricResp : metricResps) { if (metrics.contains(metricResp.getBizName())) { - if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) { + if (MetricType.isDerived(metricResp.getMetricDefineType(), + metricResp.getMetricDefineByMeasureParams())) { String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric, diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index 6583f62a6..72a5af37e 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -88,7 +88,7 @@ public class MetricServiceImplTest { new MeasureParam("s2_pv", "department='hr'"), new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); - metricReq.setTypeParams(typeParams); + metricReq.setMetricDefineByMeasureParams(typeParams); metricReq.setTags(Lists.newArrayList("核心指标")); metricReq.setRelateDimension( RelateDimension.builder().drillDownDimensions(Lists.newArrayList( @@ -119,7 +119,7 @@ public class MetricServiceImplTest { new MeasureParam("s2_pv", "department='hr'"), new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); - metricResp.setTypeParams(typeParams); + metricResp.setMetricDefineByMeasureParams(typeParams); metricResp.setTags(Lists.newArrayList("核心指标")); metricResp.setRelateDimension( RelateDimension.builder().drillDownDimensions(Lists.newArrayList( @@ -146,7 +146,7 @@ public class MetricServiceImplTest { new MeasureParam("s2_pv", "department='hr'"), new MeasureParam("s2_uv", "department='hr'"))); typeParams.setExpr("s2_pv/s2_uv"); - metricReq.setTypeParams(typeParams); + metricReq.setMetricDefineByMeasureParams(typeParams); return metricReq; } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index 41d838469..bd796d71c 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -338,7 +338,7 @@ public class ModelDemoDataLoader { "", AggOperatorEnum.SUM.getOperator()); measures.add(measure); metricTypeParams.setMeasures(measures); - metricReq.setTypeParams(metricTypeParams); + metricReq.setMetricDefineByMeasureParams(metricTypeParams); metricReq.setMetricDefineType(MetricDefineType.MEASURE); metricService.updateMetric(metricReq, user); }