From 8ed7e912219e5dcd65c462bd678e671b1e555d3e Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 10 Nov 2023 16:35:17 +0800 Subject: [PATCH] (improvement) (semantic) support metric data batch download (#358) Co-authored-by: jolunoluo --- .../supersonic/common/util/DateUtils.java | 23 ++ .../supersonic/common/util/DateUtilsTest.java | 34 +++ .../api/model/pojo/DrillDownDimension.java | 3 + .../api/model/response/MetricResp.java | 10 + .../semantic/api/query/pojo/DataDownload.java | 15 ++ .../api/query/request/BatchDownloadReq.java | 14 + .../semantic/query/rest/QueryController.java | 14 +- .../query/service/DownloadService.java | 14 + .../query/service/DownloadServiceImpl.java | 245 ++++++++++++++++++ .../semantic/query/service/QueryService.java | 4 - .../query/service/QueryServiceImpl.java | 62 ----- .../query/utils/DataTransformUtils.java | 58 +++++ .../service/DownloadServiceImplTest.java | 121 +++++++++ .../query/utils/DataTransformUtilsTest.java | 39 +++ 14 files changed, 589 insertions(+), 67 deletions(-) create mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java create mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java create mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java create mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java create mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java create mode 100644 semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java create mode 100644 semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index 434b142e1..1bbebcd8b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -6,9 +6,12 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.List; import java.util.Objects; +import com.tencent.supersonic.common.pojo.Constants; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -126,4 +129,24 @@ public class DateUtils { return !timeString.equals("00:00:00"); } + public static List getDateList(String startDateStr, String endDateStr, String period) { + LocalDate startDate = LocalDate.parse(startDateStr); + LocalDate endDate = LocalDate.parse(endDateStr); + + List datesInRange = new ArrayList<>(); + LocalDate currentDate = startDate; + + while (!currentDate.isAfter(endDate)) { + datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE)); + if (Constants.MONTH.equals(period)) { + currentDate = currentDate.plusMonths(1); + } else if (Constants.WEEK.equals(period)) { + currentDate = currentDate.plusWeeks(1); + } else { + currentDate = currentDate.plusDays(1); + } + } + return datesInRange; + } + } diff --git a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java index d15466bd3..48f6fe2c1 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java @@ -1,7 +1,11 @@ package com.tencent.supersonic.common.util; +import com.tencent.supersonic.common.pojo.Constants; +import org.assertj.core.util.Lists; import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; class DateUtilsTest { @@ -47,4 +51,34 @@ class DateUtilsTest { dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.MONTH); //Assert.assertEquals(dateStr, "2023-08-08"); } + + @Test + void testDayDateList() { + String startDate = "2023-07-29"; + String endDate = "2023-08-03"; + List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.DAY); + List expectedDateList = Lists.newArrayList("2023-07-29", "2023-07-30", + "2023-07-31", "2023-08-01", "2023-08-02", "2023-08-03"); + Assertions.assertEquals(actualDateList, expectedDateList); + } + + @Test + void testWeekDateList() { + String startDate = "2023-10-30"; + String endDate = "2023-11-13"; + List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.WEEK); + List expectedDateList = Lists.newArrayList("2023-10-30", "2023-11-06", + "2023-11-13"); + Assertions.assertEquals(actualDateList, expectedDateList); + } + + @Test + void testMonthDateList() { + String startDate = "2023-07-01"; + String endDate = "2023-10-01"; + List actualDateList = DateUtils.getDateList(startDate, endDate, Constants.MONTH); + List expectedDateList = Lists.newArrayList("2023-07-01", "2023-08-01", + "2023-09-01", "2023-10-01"); + Assertions.assertEquals(actualDateList, expectedDateList); + } } \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java index 23024ebdf..221721dfb 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/DrillDownDimension.java @@ -13,4 +13,7 @@ public class DrillDownDimension { private boolean necessary; + public DrillDownDimension(Long dimensionId) { + this.dimensionId = dimensionId; + } } \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java index 7ab64d94a..3179dcb8d 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java @@ -62,4 +62,14 @@ public class MetricResp extends SchemaItem { return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary) .map(DrillDownDimension::getDimensionId).collect(Collectors.toSet()); } + + public String getRelaDimensionIdKey() { + if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { + return ""; + } + return relateDimension.getDrillDownDimensions().stream() + .map(DrillDownDimension::getDimensionId) + .map(String::valueOf) + .collect(Collectors.joining(",")); + } } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java new file mode 100644 index 000000000..605083536 --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/DataDownload.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.semantic.api.query.pojo; + +import lombok.Builder; +import lombok.Data; +import java.util.List; + +@Data +@Builder +public class DataDownload { + + List> headers; + + List> data; + +} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java new file mode 100644 index 000000000..06919058f --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.semantic.api.query.request; + +import com.tencent.supersonic.common.pojo.DateConf; +import lombok.Data; +import java.util.List; + +@Data +public class BatchDownloadReq { + + private List metricIds; + + private DateConf dateInfo; + +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java index 4a57a12ba..b09f4217c 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.SqlParserResp; +import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; @@ -16,6 +17,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import com.tencent.supersonic.semantic.query.service.DownloadService; import com.tencent.supersonic.semantic.query.service.QueryService; import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; import java.util.List; @@ -40,6 +42,9 @@ public class QueryController { @Autowired private SemanticQueryEngine semanticQueryEngine; + @Autowired + private DownloadService downloadService; + @PostMapping("/sql") public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq, @@ -64,8 +69,15 @@ public class QueryController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - queryService.downloadByStruct(queryStructReq, user, response); + downloadService.downloadByStruct(queryStructReq, user, response); + } + @PostMapping("/download/batch") + public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + downloadService.batchDownload(batchDownloadReq, user, response); } @PostMapping("/queryStatement") diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java new file mode 100644 index 000000000..26b99d56c --- /dev/null +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.semantic.query.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import javax.servlet.http.HttpServletResponse; + +public interface DownloadService { + + void downloadByStruct(QueryStructReq queryStructReq, + User user, HttpServletResponse response) throws Exception; + + void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception; +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java new file mode 100644 index 000000000..c3d9eb508 --- /dev/null +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java @@ -0,0 +1,245 @@ +package com.tencent.supersonic.semantic.query.service; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.util.FileUtils; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.google.common.collect.Lists; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; +import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; +import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.pojo.DataDownload; +import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import com.tencent.supersonic.semantic.model.domain.ModelService; +import com.tencent.supersonic.semantic.query.utils.DataTransformUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URLEncoder; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + + +@Slf4j +@Service +public class DownloadServiceImpl implements DownloadService { + + private ModelService modelService; + + private QueryService queryService; + + public DownloadServiceImpl(ModelService modelService, QueryService queryService) { + this.modelService = modelService; + this.queryService = queryService; + } + + @Override + public void downloadByStruct(QueryStructReq queryStructReq, + User user, HttpServletResponse response) throws Exception { + QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user); + List> data = new ArrayList<>(); + List> header = org.assertj.core.util.Lists.newArrayList(); + for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + header.add(org.assertj.core.util.Lists.newArrayList(column.getName())); + } + for (Map row : queryResultWithSchemaResp.getResultList()) { + List rowData = new ArrayList<>(); + for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + rowData.add(String.valueOf(row.get(column.getNameEn()))); + } + data.add(rowData); + } + String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + File file = FileUtils.createTmpFile(fileName); + EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data); + downloadFile(response, file, fileName); + } + + private void downloadFile(HttpServletResponse response, File file, String filename) { + try { + byte[] buffer = readFileToByteArray(file); + response.reset(); + response.setCharacterEncoding("UTF-8"); + response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8")); + response.addHeader("Content-Length", "" + file.length()); + try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) { + response.setContentType("application/octet-stream"); + outputStream.write(buffer); + outputStream.flush(); + } + } catch (Exception e) { + log.error("failed to download file", e); + } + } + + private byte[] readFileToByteArray(File file) throws IOException { + try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) { + byte[] buffer = new byte[fis.available()]; + fis.read(buffer); + return buffer; + } + } + + @Override + public void batchDownload(BatchDownloadReq batchDownloadReq, User user, + HttpServletResponse response) throws Exception { + String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + File file = FileUtils.createTmpFile(fileName); + List metricIds = batchDownloadReq.getMetricIds(); + if (CollectionUtils.isEmpty(metricIds)) { + return; + } + batchDownload(batchDownloadReq, user, file); + downloadFile(response, file, fileName); + } + + public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception { + List metricIds = batchDownloadReq.getMetricIds(); + List modelSchemaRespList = modelService.fetchModelSchema(new ModelSchemaFilterReq()); + Map> metricSchemaMap = getMetricSchemaMap(modelSchemaRespList, metricIds); + Map dimensionRespMap = getDimensionMap(modelSchemaRespList); + ExcelWriter excelWriter = EasyExcel.write(file).build(); + int sheetCount = 1; + for (List metrics : metricSchemaMap.values()) { + if (CollectionUtils.isEmpty(metrics)) { + continue; + } + MetricSchemaResp metricSchemaResp = metrics.get(0); + List dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap); + for (MetricSchemaResp metric : metrics) { + DataDownload downloadData = getSingleMetricDownloadData(metric, dimensions, + batchDownloadReq.getDateInfo(), user); + WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount) + .head(downloadData.getHeaders()).build(); + excelWriter.write(downloadData.getData(), writeSheet); + } + sheetCount++; + } + excelWriter.finish(); + } + + public DataDownload getSingleMetricDownloadData(MetricSchemaResp metricSchemaResp, List dimensions, + DateConf dateConf, User user) throws Exception { + QueryResultWithSchemaResp queryResult = getQueryResult(dimensions, metricSchemaResp, dateConf, user); + List groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList()); + List dateList = getDateList(dateConf); + List> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList, + metricSchemaResp.getBizName(), groups); + List> headers = buildHeader(dimensions, dateList); + List> data = buildData(headers, getDimensionNameMap(dimensions), + dataTransformed, metricSchemaResp); + return DataDownload.builder().headers(headers).data(data).build(); + } + + private List> buildHeader(List dimensionResps, List dateList) { + List> headers = Lists.newArrayList(); + for (DimensionResp dimensionResp : dimensionResps) { + headers.add(Lists.newArrayList(dimensionResp.getName())); + } + for (String date : dateList) { + headers.add(Lists.newArrayList(date)); + } + headers.add(Lists.newArrayList("指标名")); + return headers; + } + + private List> buildData(List> headers, Map nameMap, + List> dataTransformed, MetricSchemaResp metricSchemaResp) { + List> data = Lists.newArrayList(); + for (Map map : dataTransformed) { + List row = Lists.newArrayList(); + for (List header : headers) { + String head = header.get(0); + if ("指标名".equals(head)) { + continue; + } + row.add(map.getOrDefault(nameMap.getOrDefault(head, head), "").toString()); + } + row.add(metricSchemaResp.getName()); + data.add(row); + } + return data; + } + + private List getDateList(DateConf dateConf) { + String startDateStr = dateConf.getStartDate(); + String endDateStr = dateConf.getEndDate(); + return DateUtils.getDateList(startDateStr, endDateStr, dateConf.getPeriod()); + } + + private QueryResultWithSchemaResp getQueryResult(List dimensionResps, MetricResp metricResp, + DateConf dateConf, User user) throws Exception { + QueryStructReq queryStructReq = new QueryStructReq(); + queryStructReq.setGroups(dimensionResps.stream().map(DimSchemaResp::getBizName).collect(Collectors.toList())); + queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); + Aggregator aggregator = new Aggregator(); + aggregator.setColumn(metricResp.getBizName()); + queryStructReq.setAggregators(Lists.newArrayList(aggregator)); + queryStructReq.setDateInfo(dateConf); + queryStructReq.setModelId(metricResp.getModelId()); + return queryService.queryByStruct(queryStructReq, user); + } + + private String getTimeDimension(DateConf dateConf) { + if (Constants.MONTH.equals(dateConf.getPeriod())) { + return TimeDimensionEnum.MONTH.getName(); + } else if (Constants.WEEK.equals(dateConf.getPeriod())) { + return TimeDimensionEnum.WEEK.getName(); + } else { + return TimeDimensionEnum.DAY.getName(); + } + } + + private Map> getMetricSchemaMap(List modelSchemaRespList, + List metricIds) { + return modelSchemaRespList.stream().flatMap(modelSchemaResp + -> modelSchemaResp.getMetrics().stream()) + .filter(metricSchemaResp -> metricIds.contains(metricSchemaResp.getId())) + .collect(Collectors.groupingBy(MetricSchemaResp::getRelaDimensionIdKey)); + } + + private Map getDimensionMap(List modelSchemaRespList) { + return modelSchemaRespList.stream().flatMap(modelSchemaResp + -> modelSchemaResp.getDimensions().stream()) + .collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp)); + } + + private Map getDimensionNameMap(List dimensionResps) { + return dimensionResps.stream().collect(Collectors.toMap(DimensionResp::getName, SchemaItem::getBizName)); + } + + private List getMetricRelaDimensions(MetricSchemaResp metricSchemaResp, + Map dimensionRespMap) { + return metricSchemaResp.getRelateDimension().getDrillDownDimensions() + .stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} + + diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java index 7382716e7..e6b017cba 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java @@ -12,8 +12,6 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; - -import javax.servlet.http.HttpServletResponse; import java.util.List; public interface QueryService { @@ -22,8 +20,6 @@ public interface QueryService { QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; - void downloadByStruct(QueryStructReq queryStructReq, User user, HttpServletResponse response) throws Exception; - QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index e9de34d30..2a9f4739b 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -1,14 +1,10 @@ package com.tencent.supersonic.semantic.query.service; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.util.FileUtils; import com.google.common.cache.CacheBuilder; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.common.util.ContextUtils; @@ -34,28 +30,16 @@ import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import com.tencent.supersonic.semantic.query.utils.StatUtils; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URLEncoder; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; -import org.assertj.core.util.Lists; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import javax.servlet.http.HttpServletResponse; @Service @@ -162,52 +146,6 @@ public class QueryServiceImpl implements QueryService { } } - @Override - public void downloadByStruct(QueryStructReq queryStructReq, - User user, HttpServletResponse response) throws Exception { - QueryResultWithSchemaResp queryResultWithSchemaResp = queryByStruct(queryStructReq, user); - List> data = new ArrayList<>(); - List> header = Lists.newArrayList(); - for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { - header.add(Lists.newArrayList(column.getName())); - } - for (Map row : queryResultWithSchemaResp.getResultList()) { - List rowData = new ArrayList<>(); - for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { - rowData.add(String.valueOf(row.get(column.getNameEn()))); - } - data.add(rowData); - } - String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); - File file = FileUtils.createTmpFile(fileName); - EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data); - downloadFile(response, file, fileName); - } - - private void downloadFile(HttpServletResponse response, File file, String filename) { - try { - byte[] buffer = readFileToByteArray(file); - response.reset(); - response.setCharacterEncoding("UTF-8"); - response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8")); - response.addHeader("Content-Length", "" + file.length()); - try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) { - response.setContentType("application/octet-stream"); - outputStream.write(buffer); - outputStream.flush(); - } - } catch (Exception e) { - log.error("failed to download file", e); - } - } - - private byte[] readFileToByteArray(File file) throws IOException { - try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) { - byte[] buffer = new byte[fis.available()]; - fis.read(buffer); - return buffer; - } - } @Override @DataPermission diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java new file mode 100644 index 000000000..451748a84 --- /dev/null +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java @@ -0,0 +1,58 @@ +package com.tencent.supersonic.semantic.query.utils; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class DataTransformUtils { + + public static List> transform(List> originalData, + List dateList, String metric, List groups) { + List> transposedData = new ArrayList<>(); + for (Map originalRow : originalData) { + Map transposedRow = new HashMap<>(); + for (String key : originalRow.keySet()) { + if (groups.contains(key)) { + transposedRow.put(key, originalRow.get(key)); + } + } + transposedRow.put(String.valueOf(originalRow.get(TimeDimensionEnum.DAY.getName())), + originalRow.get(metric)); + transposedData.add(transposedRow); + } + Map>> dataMerge = transposedData.stream() + .collect(Collectors.groupingBy(row -> getRowKey(row, groups))); + List> resultData = Lists.newArrayList(); + for (List> data : dataMerge.values()) { + Map rowData = new HashMap<>(); + for (Map row : data) { + for (String key : row.keySet()) { + rowData.put(key, row.get(key)); + } + } + for (String date : dateList) { + if (!rowData.containsKey(date)) { + rowData.put(date, ""); + } + } + resultData.add(rowData); + } + return resultData; + } + + private static String getRowKey(Map originalRow, List groups) { + List values = Lists.newArrayList(); + for (String key : originalRow.keySet()) { + if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) { + values.add(originalRow.get(key)); + } + } + return StringUtils.join(values, "_"); + } + +} diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java new file mode 100644 index 000000000..69d80999d --- /dev/null +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImplTest.java @@ -0,0 +1,121 @@ +package com.tencent.supersonic.semantic.query.service; + +import com.alibaba.excel.util.FileUtils; +import com.google.common.collect.Lists; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; +import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; +import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq; +import com.tencent.supersonic.semantic.model.domain.ModelService; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import java.io.File; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + + +class DownloadServiceImplTest { + @Test + void testBatchDownload() throws Exception { + ModelService modelService = Mockito.mock(ModelService.class); + QueryService queryService = Mockito.mock(QueryService.class); + when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp())); + when(queryService.queryByStruct(any(), any())).thenReturn(mockQueryResult()); + DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService); + String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + File file = FileUtils.createTmpFile(fileName); + downloadService.batchDownload(buildBatchDownloadReq(), User.getFakeUser(), file); + } + + private ModelSchemaResp mockModelSchemaResp() { + ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); + modelSchemaResp.setId(1L); + List metricResps = Lists.newArrayList(); + metricResps.add(mockMetricPv()); + metricResps.add(mockMetricUv()); + modelSchemaResp.setMetrics(metricResps); + List dimSchemaResps = Lists.newArrayList(); + dimSchemaResps.add(mockDimension(1L, "user_name", "用户名")); + dimSchemaResps.add(mockDimension(2L, "department", "部门")); + dimSchemaResps.add(mockDimension(3L, "page", "页面")); + modelSchemaResp.setDimensions(dimSchemaResps); + return modelSchemaResp; + } + + private MetricSchemaResp mockMetric(Long id, String bizName, String name, List drillDownloadDimensions) { + MetricSchemaResp metricResp = new MetricSchemaResp(); + metricResp.setId(id); + metricResp.setBizName(bizName); + metricResp.setName(name); + RelateDimension relateDimension = new RelateDimension(); + relateDimension.setDrillDownDimensions(drillDownloadDimensions.stream() + .map(DrillDownDimension::new).collect(Collectors.toList())); + metricResp.setRelateDimension(relateDimension); + return metricResp; + } + + private DimSchemaResp mockDimension(Long id, String bizName, String name) { + DimSchemaResp dimSchemaResp = new DimSchemaResp(); + dimSchemaResp.setId(id); + dimSchemaResp.setBizName(bizName); + dimSchemaResp.setName(name); + return dimSchemaResp; + } + + + private MetricSchemaResp mockMetricPv() { + return mockMetric(1L, "pv", "访问次数", Lists.newArrayList(1L, 2L)); + } + + private MetricSchemaResp mockMetricUv() { + return mockMetric(2L, "uv", "访问用户数", Lists.newArrayList(2L)); + } + + private BatchDownloadReq buildBatchDownloadReq() { + BatchDownloadReq batchDownloadReq = new BatchDownloadReq(); + batchDownloadReq.setMetricIds(Lists.newArrayList(1L)); + batchDownloadReq.setDateInfo(mockDataConf()); + return batchDownloadReq; + } + + private DateConf mockDataConf() { + DateConf dateConf = new DateConf(); + dateConf.setStartDate("2023-10-11"); + dateConf.setEndDate("2023-10-15"); + dateConf.setDateMode(DateConf.DateMode.BETWEEN); + return dateConf; + } + + private QueryResultWithSchemaResp mockQueryResult() { + QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp(); + List> resultList = Lists.newArrayList(); + resultList.add(createMap("2023-10-11", "tom", "hr", "1")); + resultList.add(createMap("2023-10-12", "alice", "sales", "2")); + resultList.add(createMap("2023-10-13", "jack", "sales", "3")); + resultList.add(createMap("2023-10-14", "luck", "market", "4")); + resultList.add(createMap("2023-10-15", "tom", "hr", "5")); + queryResultWithSchemaResp.setResultList(resultList); + return queryResultWithSchemaResp; + } + + private static Map createMap(String sysImpDate, String d1, String d2, String m1) { + Map map = new HashMap<>(); + map.put("sys_imp_date", sysImpDate); + map.put("user_name", d1); + map.put("department", d2); + map.put("pv", m1); + return map; + } + +} \ No newline at end of file diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java new file mode 100644 index 000000000..6dfa85b5c --- /dev/null +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java @@ -0,0 +1,39 @@ +package com.tencent.supersonic.semantic.query.utils; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class DataTransformUtilsTest { + + + @Test + public void testTransform() { + List> inputData = new ArrayList<>(); + inputData.add(createMap("2023/10/11", "a", "b", "1")); + inputData.add(createMap("2023/10/12", "a", "c", "2")); + inputData.add(createMap("2023/10/13", "a", "b", "3")); + inputData.add(createMap("2023/10/14", "a", "c", "4")); + inputData.add(createMap("2023/10/15", "b", "b", "5")); + List groups = Lists.newArrayList("d1", "d2"); + List dateList = Lists.newArrayList("2023/10/11", "2023/10/12", + "2023/10/13", "2023/10/14", "2023/10/15"); + String metric = "m1"; + List> resultData = DataTransformUtils.transform(inputData, dateList, metric, groups); + Assertions.assertEquals(3, resultData.size()); + } + + private static Map createMap(String sysImpDate, String d1, String d2, String m1) { + Map map = new HashMap<>(); + map.put("sys_imp_date", sysImpDate); + map.put("d1", d1); + map.put("d2", d2); + map.put("m1", m1); + return map; + } + +} \ No newline at end of file