(improvement) (semantic) support metric data batch download (#358)

Co-authored-by: jolunoluo
2025-12-11 03:58:14 +00:00 · 2023-11-10 16:35:17 +08:00
parent e537b738e4
commit 8ed7e91221
14 changed files with 589 additions and 67 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
+++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
@@ -6,9 +6,12 @@ import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.TemporalAdjusters;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 import java.util.Objects;
+import com.tencent.supersonic.common.pojo.Constants;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -126,4 +129,24 @@ public class DateUtils {
        return !timeString.equals("00:00:00");
    }

+    public static List<String> getDateList(String startDateStr, String endDateStr, String period) {
+        LocalDate startDate = LocalDate.parse(startDateStr);
+        LocalDate endDate = LocalDate.parse(endDateStr);
+
+        List<String> datesInRange = new ArrayList<>();
+        LocalDate currentDate = startDate;
+
+        while (!currentDate.isAfter(endDate)) {
+            datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
+            if (Constants.MONTH.equals(period)) {
+                currentDate = currentDate.plusMonths(1);
+            } else if (Constants.WEEK.equals(period)) {
+                currentDate = currentDate.plusWeeks(1);
+            } else {
+                currentDate = currentDate.plusDays(1);
+            }
+        }
+        return datesInRange;
+    }
+
 }
--- a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java
+++ b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java
@@ -1,7 +1,11 @@
 package com.tencent.supersonic.common.util;

+import com.tencent.supersonic.common.pojo.Constants;
+import org.assertj.core.util.Lists;
 import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import java.util.List;

 class DateUtilsTest {

@@ -47,4 +51,34 @@ class DateUtilsTest {
        dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.MONTH);
        //Assert.assertEquals(dateStr, "2023-08-08");
    }
+
+    @Test
+    void testDayDateList() {
+        String startDate = "2023-07-29";
+        String endDate = "2023-08-03";
+        List<String> actualDateList = DateUtils.getDateList(startDate, endDate, Constants.DAY);
+        List<String> expectedDateList = Lists.newArrayList("2023-07-29", "2023-07-30",
+                "2023-07-31", "2023-08-01", "2023-08-02", "2023-08-03");
+        Assertions.assertEquals(actualDateList, expectedDateList);
+    }
+
+    @Test
+    void testWeekDateList() {
+        String startDate = "2023-10-30";
+        String endDate = "2023-11-13";
+        List<String> actualDateList = DateUtils.getDateList(startDate, endDate, Constants.WEEK);
+        List<String> expectedDateList = Lists.newArrayList("2023-10-30", "2023-11-06",
+                "2023-11-13");
+        Assertions.assertEquals(actualDateList, expectedDateList);
+    }
+
+    @Test
+    void testMonthDateList() {
+        String startDate = "2023-07-01";
+        String endDate = "2023-10-01";
+        List<String> actualDateList = DateUtils.getDateList(startDate, endDate, Constants.MONTH);
+        List<String> expectedDateList = Lists.newArrayList("2023-07-01", "2023-08-01",
+                "2023-09-01", "2023-10-01");
+        Assertions.assertEquals(actualDateList, expectedDateList);
+    }
 }
--- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java
+++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java
@@ -13,4 +13,7 @@ public class DrillDownDimension {

    private boolean necessary;

+    public DrillDownDimension(Long dimensionId) {
+        this.dimensionId = dimensionId;
+    }
 }
--- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java
+++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java
@@ -62,4 +62,14 @@ public class MetricResp extends SchemaItem {
        return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary)
               .map(DrillDownDimension::getDimensionId).collect(Collectors.toSet());
    }
+
+    public String getRelaDimensionIdKey() {
+        if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
+            return "";
+        }
+        return relateDimension.getDrillDownDimensions().stream()
+                .map(DrillDownDimension::getDimensionId)
+                .map(String::valueOf)
+                .collect(Collectors.joining(","));
+    }
 }
--- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java
+++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java
@@ -0,0 +1,15 @@
+package com.tencent.supersonic.semantic.api.query.pojo;
+
+import lombok.Builder;
+import lombok.Data;
+import java.util.List;
+
+@Data
+@Builder
+public class DataDownload {
+
+    List<List<String>> headers;
+
+    List<List<String>> data;
+
+}
--- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java
+++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java
@@ -0,0 +1,14 @@
+package com.tencent.supersonic.semantic.api.query.request;
+
+import com.tencent.supersonic.common.pojo.DateConf;
+import lombok.Data;
+import java.util.List;
+
+@Data
+public class BatchDownloadReq {
+
+    private List<Long> metricIds;
+
+    private DateConf dateInfo;
+
+}
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
@@ -7,6 +7,7 @@ import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
 import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
+import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
 import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
 import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
@@ -16,6 +17,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
+import com.tencent.supersonic.semantic.query.service.DownloadService;
 import com.tencent.supersonic.semantic.query.service.QueryService;
 import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
 import java.util.List;
@@ -40,6 +42,9 @@ public class QueryController {
    @Autowired
    private SemanticQueryEngine semanticQueryEngine;

+    @Autowired
+    private DownloadService downloadService;
+

    @PostMapping("/sql")
    public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
@@ -64,8 +69,15 @@ public class QueryController {
                                HttpServletRequest request,
                                HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
-        queryService.downloadByStruct(queryStructReq, user, response);
+        downloadService.downloadByStruct(queryStructReq, user, response);
+    }

+    @PostMapping("/download/batch")
+    public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
+                                 HttpServletRequest request,
+                                 HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        downloadService.batchDownload(batchDownloadReq, user, response);
    }

    @PostMapping("/queryStatement")
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java
@@ -0,0 +1,14 @@
+package com.tencent.supersonic.semantic.query.service;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
+import javax.servlet.http.HttpServletResponse;
+
+public interface DownloadService {
+
+    void downloadByStruct(QueryStructReq queryStructReq,
+                          User user, HttpServletResponse response) throws Exception;
+
+    void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;
+}
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java
@@ -0,0 +1,245 @@
+package com.tencent.supersonic.semantic.query.service;
+
+import com.alibaba.excel.EasyExcel;
+import com.alibaba.excel.ExcelWriter;
+import com.alibaba.excel.util.FileUtils;
+import com.alibaba.excel.write.metadata.WriteSheet;
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.QueryColumn;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import com.tencent.supersonic.common.util.DateUtils;
+import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
+import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
+import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.semantic.api.model.response.MetricResp;
+import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.pojo.DataDownload;
+import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
+import com.tencent.supersonic.semantic.model.domain.ModelService;
+import com.tencent.supersonic.semantic.query.utils.DataTransformUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.stereotype.Service;
+import javax.servlet.http.HttpServletResponse;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URLEncoder;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+
+@Slf4j
+@Service
+public class DownloadServiceImpl implements DownloadService {
+
+    private ModelService modelService;
+
+    private QueryService queryService;
+
+    public DownloadServiceImpl(ModelService modelService, QueryService queryService) {
+        this.modelService = modelService;
+        this.queryService = queryService;
+    }
+
+    @Override
+    public void downloadByStruct(QueryStructReq queryStructReq,
+                                 User user, HttpServletResponse response) throws Exception {
+        QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user);
+        List<List<String>> data = new ArrayList<>();
+        List<List<String>> header = org.assertj.core.util.Lists.newArrayList();
+        for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+            header.add(org.assertj.core.util.Lists.newArrayList(column.getName()));
+        }
+        for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
+            List<String> rowData = new ArrayList<>();
+            for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+                rowData.add(String.valueOf(row.get(column.getNameEn())));
+            }
+            data.add(rowData);
+        }
+        String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
+        File file = FileUtils.createTmpFile(fileName);
+        EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data);
+        downloadFile(response, file, fileName);
+    }
+
+    private void downloadFile(HttpServletResponse response, File file, String filename) {
+        try {
+            byte[] buffer = readFileToByteArray(file);
+            response.reset();
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
+            response.addHeader("Content-Length", "" + file.length());
+            try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
+                response.setContentType("application/octet-stream");
+                outputStream.write(buffer);
+                outputStream.flush();
+            }
+        } catch (Exception e) {
+            log.error("failed to download file", e);
+        }
+    }
+
+    private byte[] readFileToByteArray(File file) throws IOException {
+        try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
+            byte[] buffer = new byte[fis.available()];
+            fis.read(buffer);
+            return buffer;
+        }
+    }
+
+    @Override
+    public void batchDownload(BatchDownloadReq batchDownloadReq, User user,
+                              HttpServletResponse response) throws Exception {
+        String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
+        File file = FileUtils.createTmpFile(fileName);
+        List<Long> metricIds = batchDownloadReq.getMetricIds();
+        if (CollectionUtils.isEmpty(metricIds)) {
+            return;
+        }
+        batchDownload(batchDownloadReq, user, file);
+        downloadFile(response, file, fileName);
+    }
+
+    public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception {
+        List<Long> metricIds = batchDownloadReq.getMetricIds();
+        List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(new ModelSchemaFilterReq());
+        Map<String, List<MetricSchemaResp>> metricSchemaMap = getMetricSchemaMap(modelSchemaRespList, metricIds);
+        Map<Long, DimSchemaResp> dimensionRespMap = getDimensionMap(modelSchemaRespList);
+        ExcelWriter excelWriter = EasyExcel.write(file).build();
+        int sheetCount = 1;
+        for (List<MetricSchemaResp> metrics : metricSchemaMap.values()) {
+            if (CollectionUtils.isEmpty(metrics)) {
+                continue;
+            }
+            MetricSchemaResp metricSchemaResp = metrics.get(0);
+            List<DimSchemaResp> dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap);
+            for (MetricSchemaResp metric : metrics) {
+                DataDownload downloadData = getSingleMetricDownloadData(metric, dimensions,
+                        batchDownloadReq.getDateInfo(), user);
+                WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
+                        .head(downloadData.getHeaders()).build();
+                excelWriter.write(downloadData.getData(), writeSheet);
+            }
+            sheetCount++;
+        }
+        excelWriter.finish();
+    }
+
+    public DataDownload getSingleMetricDownloadData(MetricSchemaResp metricSchemaResp, List<DimSchemaResp> dimensions,
+                                                    DateConf dateConf, User user) throws Exception {
+        QueryResultWithSchemaResp queryResult = getQueryResult(dimensions, metricSchemaResp, dateConf, user);
+        List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList());
+        List<String> dateList = getDateList(dateConf);
+        List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList,
+                metricSchemaResp.getBizName(), groups);
+        List<List<String>> headers = buildHeader(dimensions, dateList);
+        List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions),
+                dataTransformed, metricSchemaResp);
+        return DataDownload.builder().headers(headers).data(data).build();
+    }
+
+    private List<List<String>> buildHeader(List<DimSchemaResp> dimensionResps, List<String> dateList) {
+        List<List<String>> headers = Lists.newArrayList();
+        for (DimensionResp dimensionResp : dimensionResps) {
+            headers.add(Lists.newArrayList(dimensionResp.getName()));
+        }
+        for (String date : dateList) {
+            headers.add(Lists.newArrayList(date));
+        }
+        headers.add(Lists.newArrayList("指标名"));
+        return headers;
+    }
+
+    private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
+                                         List<Map<String, Object>> dataTransformed, MetricSchemaResp metricSchemaResp) {
+        List<List<String>> data = Lists.newArrayList();
+        for (Map<String, Object> map : dataTransformed) {
+            List<String> row = Lists.newArrayList();
+            for (List<String> header : headers) {
+                String head = header.get(0);
+                if ("指标名".equals(head)) {
+                    continue;
+                }
+                row.add(map.getOrDefault(nameMap.getOrDefault(head, head), "").toString());
+            }
+            row.add(metricSchemaResp.getName());
+            data.add(row);
+        }
+        return data;
+    }
+
+    private List<String> getDateList(DateConf dateConf) {
+        String startDateStr = dateConf.getStartDate();
+        String endDateStr = dateConf.getEndDate();
+        return DateUtils.getDateList(startDateStr, endDateStr, dateConf.getPeriod());
+    }
+
+    private QueryResultWithSchemaResp getQueryResult(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
+                                                     DateConf dateConf, User user) throws Exception {
+        QueryStructReq queryStructReq = new QueryStructReq();
+        queryStructReq.setGroups(dimensionResps.stream().map(DimSchemaResp::getBizName).collect(Collectors.toList()));
+        queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
+        Aggregator aggregator = new Aggregator();
+        aggregator.setColumn(metricResp.getBizName());
+        queryStructReq.setAggregators(Lists.newArrayList(aggregator));
+        queryStructReq.setDateInfo(dateConf);
+        queryStructReq.setModelId(metricResp.getModelId());
+        return queryService.queryByStruct(queryStructReq, user);
+    }
+
+    private String getTimeDimension(DateConf dateConf) {
+        if (Constants.MONTH.equals(dateConf.getPeriod())) {
+            return TimeDimensionEnum.MONTH.getName();
+        } else if (Constants.WEEK.equals(dateConf.getPeriod())) {
+            return TimeDimensionEnum.WEEK.getName();
+        } else {
+            return TimeDimensionEnum.DAY.getName();
+        }
+    }
+
+    private Map<String, List<MetricSchemaResp>> getMetricSchemaMap(List<ModelSchemaResp> modelSchemaRespList,
+                                                                    List<Long> metricIds) {
+        return modelSchemaRespList.stream().flatMap(modelSchemaResp
+                        -> modelSchemaResp.getMetrics().stream())
+                .filter(metricSchemaResp -> metricIds.contains(metricSchemaResp.getId()))
+                .collect(Collectors.groupingBy(MetricSchemaResp::getRelaDimensionIdKey));
+    }
+
+    private Map<Long, DimSchemaResp> getDimensionMap(List<ModelSchemaResp> modelSchemaRespList) {
+        return modelSchemaRespList.stream().flatMap(modelSchemaResp
+                        -> modelSchemaResp.getDimensions().stream())
+                .collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp));
+    }
+
+    private Map<String, String> getDimensionNameMap(List<DimSchemaResp> dimensionResps) {
+        return dimensionResps.stream().collect(Collectors.toMap(DimensionResp::getName, SchemaItem::getBizName));
+    }
+
+    private List<DimSchemaResp> getMetricRelaDimensions(MetricSchemaResp metricSchemaResp,
+                                                        Map<Long, DimSchemaResp> dimensionRespMap) {
+        return metricSchemaResp.getRelateDimension().getDrillDownDimensions()
+                .stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId()))
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
+    }
+}
+
+
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
@@ -12,8 +12,6 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
-
-import javax.servlet.http.HttpServletResponse;
 import java.util.List;

 public interface QueryService {
@@ -22,8 +20,6 @@ public interface QueryService {

    QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

-    void downloadByStruct(QueryStructReq queryStructReq, User user, HttpServletResponse response) throws Exception;
-
    QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
            throws Exception;

--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
@@ -1,14 +1,10 @@
 package com.tencent.supersonic.semantic.query.service;

-import com.alibaba.excel.EasyExcel;
-import com.alibaba.excel.util.FileUtils;
 import com.google.common.cache.CacheBuilder;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.cache.CacheUtils;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -34,28 +30,16 @@ import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
 import com.tencent.supersonic.semantic.query.utils.QueryUtils;
 import com.tencent.supersonic.semantic.query.utils.StatUtils;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URLEncoder;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.assertj.core.util.Lists;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
-import javax.servlet.http.HttpServletResponse;


@Service
@@ -162,52 +146,6 @@ public class QueryServiceImpl implements QueryService {
        }
    }

-    @Override
-    public void downloadByStruct(QueryStructReq queryStructReq,
-                                 User user, HttpServletResponse response) throws Exception {
-        QueryResultWithSchemaResp queryResultWithSchemaResp = queryByStruct(queryStructReq, user);
-        List<List<String>> data = new ArrayList<>();
-        List<List<String>> header = Lists.newArrayList();
-        for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
-            header.add(Lists.newArrayList(column.getName()));
-        }
-        for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
-            List<String> rowData = new ArrayList<>();
-            for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
-                rowData.add(String.valueOf(row.get(column.getNameEn())));
-            }
-            data.add(rowData);
-        }
-        String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
-        File file = FileUtils.createTmpFile(fileName);
-        EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data);
-        downloadFile(response, file, fileName);
-    }
-
-    private void downloadFile(HttpServletResponse response, File file, String filename) {
-        try {
-            byte[] buffer = readFileToByteArray(file);
-            response.reset();
-            response.setCharacterEncoding("UTF-8");
-            response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
-            response.addHeader("Content-Length", "" + file.length());
-            try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
-                response.setContentType("application/octet-stream");
-                outputStream.write(buffer);
-                outputStream.flush();
-            }
-        } catch (Exception e) {
-            log.error("failed to download file", e);
-        }
-    }
-
-    private byte[] readFileToByteArray(File file) throws IOException {
-        try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
-            byte[] buffer = new byte[fis.available()];
-            fis.read(buffer);
-            return buffer;
-        }
-    }

    @Override
    @DataPermission
--- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java
+++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java
@@ -0,0 +1,58 @@
+package com.tencent.supersonic.semantic.query.utils;
+
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
+import org.apache.commons.lang3.StringUtils;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class DataTransformUtils {
+
+    public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
+                                               List<String> dateList, String metric, List<String> groups) {
+        List<Map<String, Object>> transposedData = new ArrayList<>();
+        for (Map<String, Object> originalRow : originalData) {
+            Map<String, Object> transposedRow = new HashMap<>();
+            for (String key : originalRow.keySet()) {
+                if (groups.contains(key)) {
+                    transposedRow.put(key, originalRow.get(key));
+                }
+            }
+            transposedRow.put(String.valueOf(originalRow.get(TimeDimensionEnum.DAY.getName())),
+                    originalRow.get(metric));
+            transposedData.add(transposedRow);
+        }
+        Map<String, List<Map<String, Object>>> dataMerge = transposedData.stream()
+                .collect(Collectors.groupingBy(row -> getRowKey(row, groups)));
+        List<Map<String, Object>> resultData = Lists.newArrayList();
+        for (List<Map<String, Object>> data : dataMerge.values()) {
+            Map<String, Object> rowData = new HashMap<>();
+            for (Map<String, Object> row : data) {
+                for (String key : row.keySet()) {
+                    rowData.put(key, row.get(key));
+                }
+            }
+            for (String date : dateList) {
+                if (!rowData.containsKey(date)) {
+                    rowData.put(date, "");
+                }
+            }
+            resultData.add(rowData);
+        }
+        return resultData;
+    }
+
+    private static String getRowKey(Map<String, Object> originalRow, List<String> groups) {
+        List<Object> values = Lists.newArrayList();
+        for (String key : originalRow.keySet()) {
+            if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) {
+                values.add(originalRow.get(key));
+            }
+        }
+        return StringUtils.join(values, "_");
+    }
+
+}
--- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java
+++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java
@@ -0,0 +1,121 @@
+package com.tencent.supersonic.semantic.query.service;
+
+import com.alibaba.excel.util.FileUtils;
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.util.DateUtils;
+import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
+import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
+import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
+import com.tencent.supersonic.semantic.model.domain.ModelService;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import java.io.File;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+
+class DownloadServiceImplTest {
+    @Test
+    void testBatchDownload() throws Exception {
+        ModelService modelService = Mockito.mock(ModelService.class);
+        QueryService queryService = Mockito.mock(QueryService.class);
+        when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp()));
+        when(queryService.queryByStruct(any(), any())).thenReturn(mockQueryResult());
+        DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService);
+        String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
+        File file = FileUtils.createTmpFile(fileName);
+        downloadService.batchDownload(buildBatchDownloadReq(), User.getFakeUser(), file);
+    }
+
+    private ModelSchemaResp mockModelSchemaResp() {
+        ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
+        modelSchemaResp.setId(1L);
+        List<MetricSchemaResp> metricResps = Lists.newArrayList();
+        metricResps.add(mockMetricPv());
+        metricResps.add(mockMetricUv());
+        modelSchemaResp.setMetrics(metricResps);
+        List<DimSchemaResp> dimSchemaResps = Lists.newArrayList();
+        dimSchemaResps.add(mockDimension(1L, "user_name", "用户名"));
+        dimSchemaResps.add(mockDimension(2L, "department", "部门"));
+        dimSchemaResps.add(mockDimension(3L, "page", "页面"));
+        modelSchemaResp.setDimensions(dimSchemaResps);
+        return modelSchemaResp;
+    }
+
+    private MetricSchemaResp mockMetric(Long id, String bizName, String name, List<Long> drillDownloadDimensions) {
+        MetricSchemaResp metricResp = new MetricSchemaResp();
+        metricResp.setId(id);
+        metricResp.setBizName(bizName);
+        metricResp.setName(name);
+        RelateDimension relateDimension = new RelateDimension();
+        relateDimension.setDrillDownDimensions(drillDownloadDimensions.stream()
+                .map(DrillDownDimension::new).collect(Collectors.toList()));
+        metricResp.setRelateDimension(relateDimension);
+        return metricResp;
+    }
+
+    private DimSchemaResp mockDimension(Long id, String bizName, String name) {
+        DimSchemaResp dimSchemaResp = new DimSchemaResp();
+        dimSchemaResp.setId(id);
+        dimSchemaResp.setBizName(bizName);
+        dimSchemaResp.setName(name);
+        return dimSchemaResp;
+    }
+
+
+    private MetricSchemaResp mockMetricPv() {
+        return mockMetric(1L, "pv", "访问次数", Lists.newArrayList(1L, 2L));
+    }
+
+    private MetricSchemaResp mockMetricUv() {
+        return mockMetric(2L, "uv", "访问用户数", Lists.newArrayList(2L));
+    }
+
+    private BatchDownloadReq buildBatchDownloadReq() {
+        BatchDownloadReq batchDownloadReq = new BatchDownloadReq();
+        batchDownloadReq.setMetricIds(Lists.newArrayList(1L));
+        batchDownloadReq.setDateInfo(mockDataConf());
+        return batchDownloadReq;
+    }
+
+    private DateConf mockDataConf() {
+        DateConf dateConf = new DateConf();
+        dateConf.setStartDate("2023-10-11");
+        dateConf.setEndDate("2023-10-15");
+        dateConf.setDateMode(DateConf.DateMode.BETWEEN);
+        return dateConf;
+    }
+
+    private QueryResultWithSchemaResp mockQueryResult() {
+        QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
+        List<Map<String, Object>> resultList = Lists.newArrayList();
+        resultList.add(createMap("2023-10-11", "tom", "hr", "1"));
+        resultList.add(createMap("2023-10-12", "alice", "sales", "2"));
+        resultList.add(createMap("2023-10-13", "jack", "sales", "3"));
+        resultList.add(createMap("2023-10-14", "luck", "market", "4"));
+        resultList.add(createMap("2023-10-15", "tom", "hr", "5"));
+        queryResultWithSchemaResp.setResultList(resultList);
+        return queryResultWithSchemaResp;
+    }
+
+    private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {
+        Map<String, Object> map = new HashMap<>();
+        map.put("sys_imp_date", sysImpDate);
+        map.put("user_name", d1);
+        map.put("department", d2);
+        map.put("pv", m1);
+        return map;
+    }
+
+}
--- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java
+++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java
@@ -0,0 +1,39 @@
+package com.tencent.supersonic.semantic.query.utils;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class DataTransformUtilsTest {
+
+
+    @Test
+    public void testTransform() {
+        List<Map<String, Object>> inputData = new ArrayList<>();
+        inputData.add(createMap("2023/10/11", "a", "b", "1"));
+        inputData.add(createMap("2023/10/12", "a", "c", "2"));
+        inputData.add(createMap("2023/10/13", "a", "b", "3"));
+        inputData.add(createMap("2023/10/14", "a", "c", "4"));
+        inputData.add(createMap("2023/10/15", "b", "b", "5"));
+        List<String> groups = Lists.newArrayList("d1", "d2");
+        List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12",
+                "2023/10/13", "2023/10/14", "2023/10/15");
+        String metric = "m1";
+        List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList, metric, groups);
+        Assertions.assertEquals(3, resultData.size());
+    }
+
+    private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {
+        Map<String, Object> map = new HashMap<>();
+        map.put("sys_imp_date", sysImpDate);
+        map.put("d1", d1);
+        map.put("d2", d2);
+        map.put("m1", m1);
+        return map;
+    }
+
+}