(feature)add metric check parser in chat and add metric check convert in semantic, download metric data in semantic (#241)

* (improvement)(chat) add metric check parser * (improvement)(semantic) support metric data download --------- Co-authored-by: jolunoluo
2025-12-11 12:07:42 +00:00 · 2023-10-17 13:20:20 +08:00
parent 207d6cba43
commit a9bb1c1f68
16 changed files with 294 additions and 54 deletions
--- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java
+++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java
@@ -2,15 +2,20 @@ package com.tencent.supersonic.semantic.api.model.response;


 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.tencent.supersonic.common.pojo.DataFormat;
+import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
 import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
 import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
 import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import lombok.Data;
 import lombok.ToString;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;


@Data
@@ -49,4 +54,12 @@ public class MetricResp extends SchemaItem {
            tags = Arrays.asList(tag.split(","));
        }
    }
+
+    public Set<Long> getNecessaryDimensionIds() {
+        if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
+            return Sets.newHashSet();
+        }
+        return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary)
+               .map(DrillDownDimension::getDimensionId).collect(Collectors.toSet());
+    }
 }
--- a/semantic/query/pom.xml
+++ b/semantic/query/pom.xml
@@ -92,6 +92,11 @@
                </exclusion>
            </exclusions>
        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>easyexcel</artifactId>
+            <version>${easyexcel.version}</version>
+        </dependency>
    </dependencies>


--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java
@@ -0,0 +1,80 @@
+package com.tencent.supersonic.semantic.query.parser.convert;
+
+import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
+import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.semantic.api.model.response.MetricResp;
+import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.semantic.api.query.request.MetricReq;
+import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
+import com.tencent.supersonic.semantic.model.domain.Catalog;
+import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+@Component("MetricCheckConverter")
+@Slf4j
+public class MetricCheckConverter implements SemanticConverter {
+
+    @Override
+    public boolean accept(QueryStructReq queryStructCmd) {
+        if (queryStructCmd.getNativeQuery()) {
+            return false;
+        }
+        return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());
+    }
+
+    @Override
+    public void converter(Catalog catalog, QueryStructReq queryStructReq, ParseSqlReq sqlCommend,
+            MetricReq metricCommand) throws Exception {
+        Long modelId = queryStructReq.getModelId();
+        List<MetricResp> metricResps = catalog.getMetrics(modelId);
+        List<DimensionResp> dimensionResps = catalog.getDimensions(modelId);
+        Map<Long, DimensionResp> dimensionMap = dimensionResps.stream()
+                .collect(Collectors.toMap(DimensionResp::getId, d -> d));
+        List<String> metricBizNames = queryStructReq.getMetrics();
+        List<String> dimensionFilterBizNames = queryStructReq.getDimensionFilters().stream()
+                .map(Filter::getBizName).collect(Collectors.toList());
+        List<MetricResp> metricToQuery = metricResps.stream().filter(metricResp ->
+                        metricBizNames.contains(metricResp.getBizName())).collect(Collectors.toList());
+        List<Long> dimensionToFilter = dimensionResps.stream().filter(dimensionResp ->
+                dimensionFilterBizNames.contains(dimensionResp.getBizName()))
+                .map(DimensionResp::getId).collect(Collectors.toList());
+        for (MetricResp metricResp : metricToQuery) {
+            Set<Long> necessaryDimensionIds = metricResp.getNecessaryDimensionIds();
+            if (CollectionUtils.isEmpty(necessaryDimensionIds)) {
+                continue;
+            }
+            DimensionResp dimensionResp = null;
+            for (Long dimensionId : necessaryDimensionIds) {
+                dimensionResp = dimensionMap.get(dimensionId);
+                if (dimensionResp != null) {
+                    break;
+                }
+            }
+            if (dimensionResp == null) {
+                continue;
+            }
+            String message = String.format("该指标必须配合维度[%s]来进行过滤查询", dimensionResp.getName());
+            if (CollectionUtils.isEmpty(dimensionToFilter)) {
+                throw new InvalidArgumentException(message);
+            }
+            boolean flag = false;
+            for (Long dimensionId : dimensionToFilter) {
+                if (necessaryDimensionIds.contains(dimensionId)) {
+                    flag = true;
+                    break;
+                }
+            }
+            if (!flag) {
+                throw new InvalidArgumentException(message);
+            }
+        }
+    }
+}
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
@@ -59,6 +59,15 @@ public class QueryController {
        return queryService.queryByStructWithAuth(queryStructReq, user);
    }

+    @PostMapping("/download/struct")
+    public void downloadByStruct(@RequestBody QueryStructReq queryStructReq,
+                                HttpServletRequest request,
+                                HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        queryService.downloadByStruct(queryStructReq, user, response);
+
+    }
+
    @PostMapping("/queryStatement")
    public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
        return queryService.queryByQueryStatement(queryStatement);
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
@@ -13,6 +13,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;

+import javax.servlet.http.HttpServletResponse;
 import java.util.List;

 public interface QueryService {
@@ -21,6 +22,8 @@ public interface QueryService {

    QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

+    void downloadByStruct(QueryStructReq queryStructReq, User user, HttpServletResponse response) throws Exception;
+
    QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
            throws Exception;

--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
@@ -1,10 +1,14 @@
 package com.tencent.supersonic.semantic.query.service;

+import com.alibaba.excel.EasyExcel;
+import com.alibaba.excel.util.FileUtils;
 import com.google.common.cache.CacheBuilder;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.cache.CacheUtils;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -30,16 +34,28 @@ import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
 import com.tencent.supersonic.semantic.query.utils.QueryUtils;
 import com.tencent.supersonic.semantic.query.utils.StatUtils;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URLEncoder;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
+import org.assertj.core.util.Lists;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import javax.servlet.http.HttpServletResponse;


@Service
@@ -146,6 +162,53 @@ public class QueryServiceImpl implements QueryService {
        }
    }

+    @Override
+    public void downloadByStruct(QueryStructReq queryStructReq,
+                                 User user, HttpServletResponse response) throws Exception {
+        QueryResultWithSchemaResp queryResultWithSchemaResp = queryByStruct(queryStructReq, user);
+        List<List<String>> data = new ArrayList<>();
+        List<List<String>> header = Lists.newArrayList();
+        for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+            header.add(Lists.newArrayList(column.getName()));
+        }
+        for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
+            List<String> rowData = new ArrayList<>();
+            for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+                rowData.add(String.valueOf(row.get(column.getNameEn())));
+            }
+            data.add(rowData);
+        }
+        String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
+        File file = FileUtils.createTmpFile(fileName);
+        EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data);
+        downloadFile(response, file, fileName);
+    }
+
+    private void downloadFile(HttpServletResponse response, File file, String filename) {
+        try {
+            byte[] buffer = readFileToByteArray(file);
+            response.reset();
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
+            response.addHeader("Content-Length", "" + file.length());
+            try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
+                response.setContentType("application/octet-stream");
+                outputStream.write(buffer);
+                outputStream.flush();
+            }
+        } catch (Exception e) {
+            log.error("failed to download file", e);
+        }
+    }
+
+    private byte[] readFileToByteArray(File file) throws IOException {
+        try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
+            byte[] buffer = new byte[fis.available()];
+            fis.read(buffer);
+            return buffer;
+        }
+    }
+
    @Override
    @DataPermission
    @SneakyThrows
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java
@@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.query.parser.SqlParser;
 import com.tencent.supersonic.semantic.query.parser.calcite.CalciteSqlParser;
 import com.tencent.supersonic.semantic.query.parser.convert.CalculateAggConverter;
 import com.tencent.supersonic.semantic.query.parser.convert.DefaultDimValueConverter;
+import com.tencent.supersonic.semantic.query.parser.convert.MetricCheckConverter;
 import com.tencent.supersonic.semantic.query.parser.convert.MultiSourceJoin;
 import com.tencent.supersonic.semantic.query.parser.convert.ParserDefaultConverter;
 import java.util.ArrayList;
@@ -78,6 +79,7 @@ public class ComponentFactory {
    }

    private static void initSemanticConverter() {
+        semanticConverters.add(getBean("MetricCheckConverter", MetricCheckConverter.class));
        semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
        semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
        semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));