(improvement)(headless) Remove MetricCheckProcessor in chat and MetricDrillDownChecker in headless (#716)

(improvement)(headless) remove MetricCheckProcessor in chat and MetricDrillDownChecker in headless --------- Co-authored-by: jolunoluo
2025-12-11 12:07:42 +00:00 · 2024-02-04 14:28:24 +08:00
parent 4d4922d269
commit 0c4c6d83ef
18 changed files with 400 additions and 417 deletions
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java
@@ -0,0 +1,17 @@
+package com.tencent.supersonic.headless.api.pojo;
+
+
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class DefaultDisplayInfo {
+
+    //When displaying tag selection results, the information displayed by default
+    private List<Long> dimensionIds = new ArrayList<>();
+
+    private List<Long> metricIds = new ArrayList<>();
+
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java
@@ -2,15 +2,10 @@ package com.tencent.supersonic.headless.api.pojo;

 import lombok.Data;

-import java.util.ArrayList;
-import java.util.List;
-
@Data
 public class TagTypeDefaultConfig {

-    //When displaying tag selection results, the information displayed by default
-    private List<Long> dimensionIds = new ArrayList<>();
-    private List<Long> metricIds = new ArrayList<>();
+    private DefaultDisplayInfo defaultDisplayInfo;

    //default time to filter tag selection results
    private TimeDefaultConfig timeDefaultConfig;
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java
@@ -77,6 +77,13 @@ public class MetricResp extends SchemaItem {
                .collect(Collectors.joining(","));
    }

+    public List<DrillDownDimension> getDrillDownDimensions() {
+        if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
+            return Lists.newArrayList();
+        }
+        return relateDimension.getDrillDownDimensions();
+    }
+
    public String getDefaultAgg() {
        if (metricDefineByMeasureParams != null
                && CollectionUtils.isNotEmpty(metricDefineByMeasureParams.getMeasures())) {
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java
@@ -35,4 +35,24 @@ public class SemanticSchemaResp {

    }

+    public MetricSchemaResp getMetric(String bizName) {
+        return metrics.stream().filter(metric -> bizName.equalsIgnoreCase(metric.getBizName()))
+                .findFirst().orElse(null);
+    }
+
+    public MetricSchemaResp getMetric(Long id) {
+        return metrics.stream().filter(metric -> id.equals(metric.getId()))
+                .findFirst().orElse(null);
+    }
+
+    public DimSchemaResp getDimension(String bizName) {
+        return dimensions.stream().filter(dimension -> bizName.equalsIgnoreCase(dimension.getBizName()))
+                .findFirst().orElse(null);
+    }
+
+    public DimSchemaResp getDimension(Long id) {
+        return dimensions.stream().filter(dimension -> id.equals(dimension.getId()))
+                .findFirst().orElse(null);
+    }
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
@@ -0,0 +1,150 @@
+package com.tencent.supersonic.headless.server.aspect;
+
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
+import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Aspect
+@Component
+@Slf4j
+public class MetricDrillDownChecker {
+
+    @Around("execution(* com.tencent.supersonic.headless.core.parser.QueryParser.parse(..))")
+    public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
+        Object[] objects = joinPoint.getArgs();
+        QueryStatement queryStatement = (QueryStatement) objects[0];
+        if (queryStatement.getParseSqlReq() == null) {
+            return joinPoint.proceed();
+        }
+        checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getParseSqlReq().getSql());
+        return joinPoint.proceed();
+    }
+
+    public void checkQuery(SemanticSchemaResp semanticSchemaResp, String sql) {
+        List<String> groupByFields = SqlSelectHelper.getGroupByFields(sql);
+        List<String> metricFields = SqlSelectHelper.getAggregateAsFields(sql);
+        List<String> whereFields = SqlSelectHelper.getWhereFields(sql);
+        List<String> dimensionFields = getDimensionFields(groupByFields, whereFields);
+        if (CollectionUtils.isEmpty(metricFields) || StringUtils.isBlank(sql)) {
+            return;
+        }
+        for (String metricName : metricFields) {
+            MetricSchemaResp metric = semanticSchemaResp.getMetric(metricName);
+            List<DimensionResp> necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp);
+            List<DimensionResp> dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields);
+            if (!CollectionUtils.isEmpty(dimensionsMissing)) {
+                String errMsg = String.format("指标:%s 缺失必要维度:%s", metric.getName(),
+                        dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList()));
+                throw new InvalidArgumentException(errMsg);
+            }
+        }
+        for (String dimensionBizName : groupByFields) {
+            if (TimeDimensionEnum.containsTimeDimension(dimensionBizName)) {
+                continue;
+            }
+            List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
+            if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
+                DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);
+                String errMsg = String.format("维度:%s, 不在当前查询指标的下钻维度配置中, 请检查", dimSchemaResp.getName());
+                throw new InvalidArgumentException(errMsg);
+            }
+        }
+    }
+
+    /**
+     * To check whether the dimension bound to the metric exists,
+     * eg: metric like UV is calculated in a certain dimension, it cannot be used on other dimensions.
+     */
+    private List<DimensionResp> getNecessaryDimensionMissing(List<DimensionResp> necessaryDimensions,
+                                                             List<String> dimensionFields) {
+        return necessaryDimensions.stream()
+                .filter(dimension -> !dimensionFields.contains(dimension.getBizName()))
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * To check whether the dimension can drill down the metric,
+     * eg: some descriptive dimensions are not suitable as drill-down dimensions
+     */
+    private boolean checkDrillDownDimension(String dimensionName,
+                                            List<MetricResp> metricResps,
+                                            SemanticSchemaResp semanticSchemaResp) {
+        if (CollectionUtils.isEmpty(metricResps)) {
+            return true;
+        }
+        List<String> relateDimensions = metricResps.stream()
+                .filter(metric -> !CollectionUtils.isEmpty(metric.getDrillDownDimensions()))
+                .map(metric -> metric.getDrillDownDimensions().stream()
+                        .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()))
+                .flatMap(Collection::stream)
+                .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
+        //if no metric has drill down dimension, return true
+        if (CollectionUtils.isEmpty(relateDimensions)) {
+            return true;
+        }
+        //if this dimension not in relate drill-down dimensions, return false
+        return relateDimensions.contains(dimensionName);
+    }
+
+    private List<DimensionResp> getNecessaryDimensions(MetricSchemaResp metric, SemanticSchemaResp semanticSchemaResp) {
+        if (metric == null) {
+            return Lists.newArrayList();
+        }
+        List<DrillDownDimension> drillDownDimensions = metric.getDrillDownDimensions();
+        if (CollectionUtils.isEmpty(drillDownDimensions)) {
+            return Lists.newArrayList();
+        }
+        return drillDownDimensions.stream()
+                .filter(DrillDownDimension::isNecessary).map(DrillDownDimension::getDimensionId)
+                .map(semanticSchemaResp::getDimension)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
+    }
+
+    private List<String> getDimensionFields(List<String> groupByFields, List<String> whereFields) {
+        List<String> dimensionFields = Lists.newArrayList();
+        if (!CollectionUtils.isEmpty(groupByFields)) {
+            dimensionFields.addAll(groupByFields);
+        }
+        if (!CollectionUtils.isEmpty(whereFields)) {
+            dimensionFields.addAll(whereFields);
+        }
+        return dimensionFields;
+    }
+
+    private List<MetricResp> getMetrics(List<String> metricFields, SemanticSchemaResp semanticSchemaResp) {
+        return semanticSchemaResp.getMetrics().stream()
+                .filter(metricSchemaResp -> metricFields.contains(metricSchemaResp.getBizName()))
+                .collect(Collectors.toList());
+    }
+
+    private String convertDimensionIdToBizName(Long id, SemanticSchemaResp semanticSchemaResp) {
+        DimSchemaResp dimension = semanticSchemaResp.getDimension(id);
+        if (dimension == null) {
+            return null;
+        }
+        return dimension.getBizName();
+    }
+
+}
--- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java
+++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownCheckerTest.java
@@ -0,0 +1,88 @@
+package com.tencent.supersonic.headless.server.aspect;
+
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
+import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.server.utils.DataUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import java.util.List;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+
+@Slf4j
+public class MetricDrillDownCheckerTest {
+
+    @Test
+    void test_groupBy_in_drillDownDimension() {
+        MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker();
+        String sql = "select user_name, sum(pv) from t_1 group by user_name";
+        SemanticSchemaResp semanticSchemaResp = mockModelSchemaResp();
+        metricDrillDownChecker.checkQuery(semanticSchemaResp, sql);
+    }
+
+    @Test
+    void test_groupBy_not_in_drillDownDimension() {
+        MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker();
+        String sql = "select page, sum(pv) from t_1 group by page";
+        SemanticSchemaResp semanticSchemaResp = mockModelSchemaResp();
+        assertThrows(InvalidArgumentException.class,
+                () -> metricDrillDownChecker.checkQuery(semanticSchemaResp, sql));
+    }
+
+    @Test
+    void test_groupBy_not_in_necessary_dimension() {
+        MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker();
+        String sql = "select user_name, count(distinct uv) from t_1 group by user_name";
+        SemanticSchemaResp semanticSchemaResp = mockModelSchemaResp();
+        assertThrows(InvalidArgumentException.class,
+                () -> metricDrillDownChecker.checkQuery(semanticSchemaResp, sql));
+    }
+
+    @Test
+    void test_groupBy_no_necessary_dimension_setting() {
+        MetricDrillDownChecker metricDrillDownChecker = new MetricDrillDownChecker();
+        String sql = "select user_name, page, count(distinct uv) from t_1 group by user_name,page";
+        SemanticSchemaResp semanticSchemaResp = mockModelSchemaNoDimensionSetting();
+        metricDrillDownChecker.checkQuery(semanticSchemaResp, sql);
+    }
+
+    private SemanticSchemaResp mockModelSchemaResp() {
+        SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
+        semanticSchemaResp.setMetrics(mockMetrics());
+        semanticSchemaResp.setDimensions(mockDimensions());
+        return semanticSchemaResp;
+    }
+
+    private SemanticSchemaResp mockModelSchemaNoDimensionSetting() {
+        SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
+        List<MetricSchemaResp> metricSchemaResps = Lists.newArrayList(mockMetricsNoDrillDownSetting());
+        semanticSchemaResp.setMetrics(metricSchemaResps);
+        semanticSchemaResp.setDimensions(mockDimensions());
+        return semanticSchemaResp;
+    }
+
+    private List<DimSchemaResp> mockDimensions() {
+        return Lists.newArrayList(DataUtils.mockDimension(1L, "user_name", "用户名"),
+                DataUtils.mockDimension(2L, "department", "部门"),
+                DataUtils.mockDimension(3L, "page", "页面"));
+    }
+
+    private List<MetricSchemaResp> mockMetrics() {
+        return Lists.newArrayList(
+                DataUtils.mockMetric(1L, "pv", "访问次数",
+                        Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(2L))),
+                DataUtils.mockMetric(2L, "uv", "访问用户数",
+                        Lists.newArrayList(new DrillDownDimension(2L, true))));
+    }
+
+    private List<MetricSchemaResp> mockMetricsNoDrillDownSetting() {
+        return Lists.newArrayList(
+                DataUtils.mockMetric(1L, "pv", Lists.newArrayList()),
+                DataUtils.mockMetric(2L, "uv", Lists.newArrayList()));
+    }
+
+}
--- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java
+++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataUtils.java
@@ -0,0 +1,45 @@
+package com.tencent.supersonic.headless.server.utils;
+
+import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.RelateDimension;
+import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
+
+import java.util.List;
+
+public class DataUtils {
+
+    public static DimSchemaResp mockDimension(Long id, String bizName, String name) {
+        DimSchemaResp dimSchemaResp = new DimSchemaResp();
+        dimSchemaResp.setId(id);
+        dimSchemaResp.setBizName(bizName);
+        dimSchemaResp.setName(name);
+        return dimSchemaResp;
+    }
+
+    public static MetricSchemaResp mockMetric(Long id, String bizName) {
+        MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
+        metricSchemaResp.setId(id);
+        metricSchemaResp.setBizName(bizName);
+        RelateDimension relateDimension = new RelateDimension();
+        metricSchemaResp.setRelateDimension(relateDimension);
+        return metricSchemaResp;
+    }
+
+    public static MetricSchemaResp mockMetric(Long id, String bizName, String name,
+                                        List<DrillDownDimension> drillDownDimensions) {
+        MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
+        metricSchemaResp.setId(id);
+        metricSchemaResp.setName(name);
+        metricSchemaResp.setBizName(bizName);
+        metricSchemaResp.setRelateDimension(RelateDimension.builder()
+                .drillDownDimensions(drillDownDimensions).build());
+        return metricSchemaResp;
+    }
+
+    public static MetricSchemaResp mockMetric(Long id, String bizName,
+                                              List<DrillDownDimension> drillDownDimensions) {
+        return mockMetric(id, bizName, null, drillDownDimensions);
+    }
+
+}