topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(semantic) Metric homepage download supports choosing whether to transform (#477)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-12-06 18:02:20 +08:00
committed by GitHub
parent ed0f856438
commit 9679169e6f
11 changed files with 172 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -8,21 +8,19 @@ import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.DownloadService;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -31,6 +29,9 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/query")
@@ -64,11 +65,11 @@ public class QueryController {
}
@PostMapping("/download/struct")
public void downloadByStruct(@RequestBody QueryStructReq queryStructReq,
public void downloadByStruct(@RequestBody DownloadStructReq downloadStructReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
downloadService.downloadByStruct(queryStructReq, user, response);
downloadService.downloadByStruct(downloadStructReq, user, response);
}
@PostMapping("/download/batch")

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
import javax.servlet.http.HttpServletResponse;
public interface DownloadService {
void downloadByStruct(QueryStructReq queryStructReq,
void downloadByStruct(DownloadStructReq downloadStructReq,
User user, HttpServletResponse response) throws Exception;
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;

185
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
@@ -22,7 +22,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.DataDownload;
import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.utils.DataTransformUtils;
import lombok.extern.slf4j.Slf4j;
@@ -51,6 +51,10 @@ import java.util.stream.Collectors;
@Service
public class DownloadServiceImpl implements DownloadService {
private static final String internMetricCol = "指标名称";
private static final long downloadSize = 10000;
private ModelService modelService;
private QueryService queryService;
@@ -61,50 +65,20 @@ public class DownloadServiceImpl implements DownloadService {
}
@Override
public void downloadByStruct(QueryStructReq queryStructReq,
public void downloadByStruct(DownloadStructReq downloadStructReq,
User user, HttpServletResponse response) throws Exception {
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user);
List<List<String>> data = new ArrayList<>();
List<List<String>> header = org.assertj.core.util.Lists.newArrayList();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
header.add(Lists.newArrayList(column.getName()));
}
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
List<String> rowData = new ArrayList<>();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn())));
}
data.add(rowData);
}
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
File file = FileUtils.createTmpFile(fileName);
EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data);
downloadFile(response, file, fileName);
}
private void downloadFile(HttpServletResponse response, File file, String filename) {
try {
byte[] buffer = readFileToByteArray(file);
response.reset();
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
response.addHeader("Content-Length", "" + file.length());
try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
response.setContentType("application/octet-stream");
outputStream.write(buffer);
outputStream.flush();
}
} catch (Exception e) {
log.error("failed to download file", e);
}
}
private byte[] readFileToByteArray(File file) throws IOException {
try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
byte[] buffer = new byte[fis.available()];
fis.read(buffer);
return buffer;
QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResultWithSchemaResp, downloadStructReq);
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
} catch (RuntimeException e) {
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
.doWrite(buildErrMessageData(e.getMessage()));
return;
}
downloadFile(response, file, fileName);
}
@Override
@@ -135,11 +109,12 @@ public class DownloadServiceImpl implements DownloadService {
List<DimSchemaResp> dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap);
for (MetricSchemaResp metric : metrics) {
try {
DataDownload downloadData = getSingleMetricDownloadData(metric, dimensions,
batchDownloadReq.getDateInfo(), user);
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(downloadData.getHeaders()).build();
excelWriter.write(downloadData.getData(), writeSheet);
.head(dataDownload.getHeaders()).build();
excelWriter.write(dataDownload.getData(), writeSheet);
} catch (RuntimeException e) {
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
.doWrite(buildErrMessageData(e.getMessage()));
@@ -163,39 +138,49 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
public DataDownload getSingleMetricDownloadData(MetricSchemaResp metricSchemaResp, List<DimSchemaResp> dimensions,
DateConf dateConf, User user) throws Exception {
QueryResultWithSchemaResp queryResult = getQueryResult(dimensions, metricSchemaResp, dateConf, user);
List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList());
List<String> dateList = getDateList(dateConf);
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList,
metricSchemaResp.getBizName(), groups, dateConf);
List<List<String>> headers = buildHeader(dimensions, dateList);
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions),
dataTransformed, metricSchemaResp);
return DataDownload.builder().headers(headers).data(data).build();
private List<List<String>> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) {
List<List<String>> header = Lists.newArrayList();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
header.add(Lists.newArrayList(column.getName()));
}
return header;
}
private List<List<String>> buildHeader(List<DimSchemaResp> dimensionResps, List<String> dateList) {
private List<List<String>> buildHeader(List<QueryColumn> queryColumns, List<String> dateList) {
List<List<String>> headers = Lists.newArrayList();
for (DimensionResp dimensionResp : dimensionResps) {
headers.add(Lists.newArrayList(dimensionResp.getName()));
for (QueryColumn queryColumn : queryColumns) {
if (SemanticTypeEnum.DATE.name().equals(queryColumn.getShowType())) {
continue;
}
headers.add(Lists.newArrayList(queryColumn.getName()));
}
for (String date : dateList) {
headers.add(Lists.newArrayList(date));
}
headers.add(Lists.newArrayList("指标名"));
headers.add(Lists.newArrayList(internMetricCol));
return headers;
}
private List<List<String>> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) {
List<List<String>> data = new ArrayList<>();
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
List<String> rowData = new ArrayList<>();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn())));
}
data.add(rowData);
}
return data;
}
private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
List<Map<String, Object>> dataTransformed, MetricSchemaResp metricSchemaResp) {
List<Map<String, Object>> dataTransformed, String metricName) {
List<List<String>> data = Lists.newArrayList();
for (Map<String, Object> map : dataTransformed) {
List<String> row = Lists.newArrayList();
for (List<String> header : headers) {
String head = header.get(0);
if ("指标名".equals(head)) {
if (internMetricCol.equals(head)) {
continue;
}
Object object = map.getOrDefault(nameMap.getOrDefault(head, head), "");
@@ -205,32 +190,48 @@ public class DownloadServiceImpl implements DownloadService {
row.add(String.valueOf(object));
}
}
row.add(metricSchemaResp.getName());
row.add(metricName);
data.add(row);
}
return data;
}
private List<String> getDateList(DateConf dateConf) {
String startDateStr = dateConf.getStartDate();
String endDateStr = dateConf.getEndDate();
return DateUtils.getDateList(startDateStr, endDateStr, dateConf.getPeriod());
private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) {
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {
QueryColumn metric = metricColumns.get(0);
List<String> groups = downloadStructReq.getGroups();
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(),
metric.getNameEn(), groups, downloadStructReq.getDateInfo());
List<List<String>> headers = buildHeader(dimensionColumns, downloadStructReq.getDateInfo().getDateList());
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
dataTransformed, metric.getName());
return DataDownload.builder().headers(headers).data(data).build();
} else {
List<List<String>> data = buildData(queryResult);
List<List<String>> header = buildHeader(queryResult);
return DataDownload.builder().data(data).headers(header).build();
}
}
private QueryResultWithSchemaResp getQueryResult(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
DateConf dateConf, User user) throws Exception {
private DownloadStructReq buildDownloadStructReq(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
BatchDownloadReq batchDownloadReq) {
DateConf dateConf = batchDownloadReq.getDateInfo();
Set<Long> modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet());
modelIds.add(metricResp.getModelId());
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(dimensionResps.stream().map(DimSchemaResp::getBizName).collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
DownloadStructReq downloadStructReq = new DownloadStructReq();
downloadStructReq.setGroups(dimensionResps.stream()
.map(DimSchemaResp::getBizName).collect(Collectors.toList()));
downloadStructReq.getGroups().add(0, getTimeDimension(dateConf));
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelIds(modelIds);
queryStructReq.setLimit(10000L);
return queryService.queryByStructWithAuth(queryStructReq, user);
downloadStructReq.setAggregators(Lists.newArrayList(aggregator));
downloadStructReq.setDateInfo(dateConf);
downloadStructReq.setModelIds(modelIds);
downloadStructReq.setLimit(downloadSize);
downloadStructReq.setIsTransform(batchDownloadReq.isTransform());
return downloadStructReq;
}
private String getTimeDimension(DateConf dateConf) {
@@ -257,8 +258,8 @@ public class DownloadServiceImpl implements DownloadService {
.collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp));
}
private Map<String, String> getDimensionNameMap(List<DimSchemaResp> dimensionResps) {
return dimensionResps.stream().collect(Collectors.toMap(DimensionResp::getName, SchemaItem::getBizName));
private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
}
private List<DimSchemaResp> getMetricRelaDimensions(MetricSchemaResp metricSchemaResp,
@@ -272,6 +273,32 @@ public class DownloadServiceImpl implements DownloadService {
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
private void downloadFile(HttpServletResponse response, File file, String filename) {
try {
byte[] buffer = readFileToByteArray(file);
response.reset();
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
response.addHeader("Content-Length", "" + file.length());
try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
response.setContentType("application/octet-stream");
outputStream.write(buffer);
outputStream.flush();
}
} catch (Exception e) {
log.error("failed to download file", e);
}
}
private byte[] readFileToByteArray(File file) throws IOException {
try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
byte[] buffer = new byte[fis.available()];
fis.read(buffer);
return buffer;
}
}
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java
View File

@@ -13,9 +13,9 @@ import java.util.stream.Collectors;
public class DataTransformUtils {
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
List<String> dateList, String metric, List<String> groups,
DateConf dateConf) {
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData, String metric,
List<String> groups, DateConf dateConf) {
List<String> dateList = dateConf.getDateList();
List<Map<String, Object>> transposedData = new ArrayList<>();
for (Map<String, Object> originalRow : originalData) {
Map<String, Object> transposedRow = new HashMap<>();

4
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java
View File

@@ -20,10 +20,8 @@ class DataTransformUtilsTest {
inputData.add(createMap("2023/10/14", "a", "c", "4"));
inputData.add(createMap("2023/10/15", "b", "b", "5"));
List<String> groups = Lists.newArrayList("d1", "d2");
List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12",
"2023/10/13", "2023/10/14", "2023/10/15");
String metric = "m1";
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList,
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData,
metric, groups, new DateConf());
Assertions.assertEquals(3, resultData.size());
}