(improvement)(semantic) Metric homepage download supports choosing whether to transform (#477)

Co-authored-by: jolunoluo

chat/core/src/main/resources/mapper/custom/ShowCaseCustomMapper.xml

@@ -61,16 +61,15 @@
             from (
                select *
                from s2_chat_query
-               where query_state = 1 and agent_id = ${agentId} and (score is null or score > 1)
+               where query_state = 1 and agent_id = ${agentId} and score = 5
                <if test="userName != null and userName != ''">
                      and user_name = #{userName}
                </if>
-               order by score, chat_id desc
+               order by chat_id desc
             ) a
             limit #{start}, #{limit}
         ) q2 on q1.chat_id = q2.chat_id
-        where agent_id = ${agentId} and (score is null or score > 1)
-        order by score desc
+        where agent_id = ${agentId} and score = 5
     </select>
 
 </mapper>

common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.common.pojo;
 
-import static java.time.LocalDate.now;
+import com.tencent.supersonic.common.util.DateUtils;
+import lombok.Data;
+import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
-import lombok.Data;
+import static java.time.LocalDate.now;
 
 @Data
 public class DateConf {
@@ -44,6 +46,15 @@ public class DateConf {
 
     private boolean isInherited;
 
+    public List<String> getDateList() {
+        if (!CollectionUtils.isEmpty(dateList)) {
+            return dateList;
+        }
+        String startDateStr = getStartDate();
+        String endDateStr = getEndDate();
+        return DateUtils.getDateList(startDateStr, endDateStr, getPeriod());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

launchers/standalone/src/test/resources/application-local.yaml

@@ -4,8 +4,6 @@ spring:
       path: /h2-console/chat
       # enabled web
       enabled: true
-    demo:
-      enabled: true
   datasource:
     driver-class-name: org.h2.Driver
     schema: classpath:db/schema-h2.sql
@@ -14,6 +12,9 @@ spring:
     username: root
     password: semantic
 
+demo:
+  enabled: true
+
 server:
   port: 9080
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/QueryResultWithSchemaResp.java

@@ -1,20 +1,34 @@
 package com.tencent.supersonic.semantic.api.model.response;
 
 
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.QueryAuthorization;
 import com.tencent.supersonic.common.pojo.QueryColumn;
+import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
 import com.tencent.supersonic.semantic.api.model.pojo.QueryResult;
-import java.util.List;
-import java.util.Map;
-
 import lombok.Data;
 import lombok.ToString;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 @Data
 @ToString
 public class QueryResultWithSchemaResp extends QueryResult<Map<String, Object>> {
 
-    List<QueryColumn> columns;
+    List<QueryColumn> columns = Lists.newArrayList();
     String sql;
     QueryAuthorization queryAuthorization;
+
+    public List<QueryColumn> getMetricColumns() {
+        return columns.stream()
+                .filter(queryColumn -> SemanticTypeEnum.NUMBER.name().equals(queryColumn.getShowType()))
+                .collect(Collectors.toList());
+    }
+
+    public List<QueryColumn> getDimensionColumns() {
+        return columns.stream()
+                .filter(queryColumn -> !SemanticTypeEnum.NUMBER.name().equals(queryColumn.getShowType()))
+                .collect(Collectors.toList());
+    }
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/BatchDownloadReq.java

@@ -11,4 +11,6 @@ public class BatchDownloadReq {
 
     private DateConf dateInfo;
 
+    private boolean isTransform = true;
+
 }

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/DownloadStructReq.java

@@ -0,0 +1,14 @@
+package com.tencent.supersonic.semantic.api.query.request;
+
+import lombok.Data;
+
+@Data
+public class DownloadStructReq extends QueryStructReq {
+
+    private boolean isTransform;
+
+    public void setIsTransform(boolean isTransform) {
+        this.isTransform = isTransform;
+    }
+
+}

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java

@@ -8,21 +8,19 @@ import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
 import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
+import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
 import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
 import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
-import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
 import com.tencent.supersonic.semantic.query.service.DownloadService;
 import com.tencent.supersonic.semantic.query.service.QueryService;
 import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
-import java.util.List;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,6 +29,9 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.util.List;
 
 @RestController
 @RequestMapping("/api/semantic/query")
@@ -64,11 +65,11 @@ public class QueryController {
     }
 
     @PostMapping("/download/struct")
-    public void downloadByStruct(@RequestBody QueryStructReq queryStructReq,
+    public void downloadByStruct(@RequestBody DownloadStructReq downloadStructReq,
                                 HttpServletRequest request,
                                 HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        downloadService.downloadByStruct(queryStructReq, user, response);
+        downloadService.downloadByStruct(downloadStructReq, user, response);
     }
 
     @PostMapping("/download/batch")

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadService.java

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
-import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
+import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
 import javax.servlet.http.HttpServletResponse;
 
 public interface DownloadService {
 
-    void downloadByStruct(QueryStructReq queryStructReq,
+    void downloadByStruct(DownloadStructReq downloadStructReq,
                           User user, HttpServletResponse response) throws Exception;
 
     void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.util.DateUtils;
-import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
+import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
@@ -22,7 +22,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.query.pojo.DataDownload;
 import com.tencent.supersonic.semantic.api.query.request.BatchDownloadReq;
-import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
+import com.tencent.supersonic.semantic.api.query.request.DownloadStructReq;
 import com.tencent.supersonic.semantic.model.domain.ModelService;
 import com.tencent.supersonic.semantic.query.utils.DataTransformUtils;
 import lombok.extern.slf4j.Slf4j;
@@ -51,6 +51,10 @@ import java.util.stream.Collectors;
 @Service
 public class DownloadServiceImpl implements DownloadService {
 
+    private static final String internMetricCol = "指标名称";
+
+    private static final long downloadSize = 10000;
+
     private ModelService modelService;
 
     private QueryService queryService;
@@ -61,50 +65,20 @@ public class DownloadServiceImpl implements DownloadService {
     }
 
     @Override
-    public void downloadByStruct(QueryStructReq queryStructReq,
+    public void downloadByStruct(DownloadStructReq downloadStructReq,
                                  User user, HttpServletResponse response) throws Exception {
-        QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user);
-        List<List<String>> data = new ArrayList<>();
-        List<List<String>> header = org.assertj.core.util.Lists.newArrayList();
-        for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
-            header.add(Lists.newArrayList(column.getName()));
-        }
-        for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
-            List<String> rowData = new ArrayList<>();
-            for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
-                rowData.add(String.valueOf(row.get(column.getNameEn())));
-            }
-            data.add(rowData);
-        }
         String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
         File file = FileUtils.createTmpFile(fileName);
-        EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data);
-        downloadFile(response, file, fileName);
-    }
-
-    private void downloadFile(HttpServletResponse response, File file, String filename) {
         try {
-            byte[] buffer = readFileToByteArray(file);
-            response.reset();
-            response.setCharacterEncoding("UTF-8");
-            response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
-            response.addHeader("Content-Length", "" + file.length());
-            try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
-                response.setContentType("application/octet-stream");
-                outputStream.write(buffer);
-                outputStream.flush();
-            }
-        } catch (Exception e) {
-            log.error("failed to download file", e);
-        }
-    }
-
-    private byte[] readFileToByteArray(File file) throws IOException {
-        try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
-            byte[] buffer = new byte[fis.available()];
-            fis.read(buffer);
-            return buffer;
+            QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(downloadStructReq, user);
+            DataDownload dataDownload = buildDataDownload(queryResultWithSchemaResp, downloadStructReq);
+            EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
+        } catch (RuntimeException e) {
+            EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
+                    .doWrite(buildErrMessageData(e.getMessage()));
+            return;
         }
+        downloadFile(response, file, fileName);
     }
 
     @Override
@@ -135,11 +109,12 @@ public class DownloadServiceImpl implements DownloadService {
             List<DimSchemaResp> dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap);
             for (MetricSchemaResp metric : metrics) {
                 try {
-                    DataDownload downloadData = getSingleMetricDownloadData(metric, dimensions,
-                            batchDownloadReq.getDateInfo(), user);
+                    DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
+                    QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
+                    DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
                     WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
-                            .head(downloadData.getHeaders()).build();
-                    excelWriter.write(downloadData.getData(), writeSheet);
+                            .head(dataDownload.getHeaders()).build();
+                    excelWriter.write(dataDownload.getData(), writeSheet);
                 } catch (RuntimeException e) {
                     EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
                             .doWrite(buildErrMessageData(e.getMessage()));
@@ -163,39 +138,49 @@ public class DownloadServiceImpl implements DownloadService {
         return data;
     }
 
-    public DataDownload getSingleMetricDownloadData(MetricSchemaResp metricSchemaResp, List<DimSchemaResp> dimensions,
-                                                    DateConf dateConf, User user) throws Exception {
-        QueryResultWithSchemaResp queryResult = getQueryResult(dimensions, metricSchemaResp, dateConf, user);
-        List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList());
-        List<String> dateList = getDateList(dateConf);
-        List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList,
-                metricSchemaResp.getBizName(), groups, dateConf);
-        List<List<String>> headers = buildHeader(dimensions, dateList);
-        List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions),
-                dataTransformed, metricSchemaResp);
-        return DataDownload.builder().headers(headers).data(data).build();
+    private List<List<String>> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) {
+        List<List<String>> header = Lists.newArrayList();
+        for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+            header.add(Lists.newArrayList(column.getName()));
+        }
+        return header;
     }
 
-    private List<List<String>> buildHeader(List<DimSchemaResp> dimensionResps, List<String> dateList) {
+    private List<List<String>> buildHeader(List<QueryColumn> queryColumns, List<String> dateList) {
         List<List<String>> headers = Lists.newArrayList();
-        for (DimensionResp dimensionResp : dimensionResps) {
-            headers.add(Lists.newArrayList(dimensionResp.getName()));
+        for (QueryColumn queryColumn : queryColumns) {
+            if (SemanticTypeEnum.DATE.name().equals(queryColumn.getShowType())) {
+                continue;
+            }
+            headers.add(Lists.newArrayList(queryColumn.getName()));
         }
         for (String date : dateList) {
             headers.add(Lists.newArrayList(date));
         }
-        headers.add(Lists.newArrayList("指标名"));
+        headers.add(Lists.newArrayList(internMetricCol));
         return headers;
     }
 
+    private List<List<String>> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) {
+        List<List<String>> data = new ArrayList<>();
+        for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
+            List<String> rowData = new ArrayList<>();
+            for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
+                rowData.add(String.valueOf(row.get(column.getNameEn())));
+            }
+            data.add(rowData);
+        }
+        return data;
+    }
+
     private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
-                                         List<Map<String, Object>> dataTransformed, MetricSchemaResp metricSchemaResp) {
+                                         List<Map<String, Object>> dataTransformed, String metricName) {
         List<List<String>> data = Lists.newArrayList();
         for (Map<String, Object> map : dataTransformed) {
             List<String> row = Lists.newArrayList();
             for (List<String> header : headers) {
                 String head = header.get(0);
-                if ("指标名".equals(head)) {
+                if (internMetricCol.equals(head)) {
                     continue;
                 }
                 Object object = map.getOrDefault(nameMap.getOrDefault(head, head), "");
@@ -205,32 +190,48 @@ public class DownloadServiceImpl implements DownloadService {
                     row.add(String.valueOf(object));
                 }
             }
-            row.add(metricSchemaResp.getName());
+            row.add(metricName);
             data.add(row);
         }
         return data;
     }
 
-    private List<String> getDateList(DateConf dateConf) {
-        String startDateStr = dateConf.getStartDate();
-        String endDateStr = dateConf.getEndDate();
-        return DateUtils.getDateList(startDateStr, endDateStr, dateConf.getPeriod());
+    private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) {
+        List<QueryColumn> metricColumns = queryResult.getMetricColumns();
+        List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
+        if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {
+            QueryColumn metric = metricColumns.get(0);
+            List<String> groups = downloadStructReq.getGroups();
+            List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(),
+                    metric.getNameEn(), groups, downloadStructReq.getDateInfo());
+            List<List<String>> headers = buildHeader(dimensionColumns, downloadStructReq.getDateInfo().getDateList());
+            List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
+                    dataTransformed, metric.getName());
+            return DataDownload.builder().headers(headers).data(data).build();
+        } else {
+            List<List<String>> data = buildData(queryResult);
+            List<List<String>> header = buildHeader(queryResult);
+            return DataDownload.builder().data(data).headers(header).build();
+        }
     }
 
-    private QueryResultWithSchemaResp getQueryResult(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
-                                                     DateConf dateConf, User user) throws Exception {
+    private DownloadStructReq buildDownloadStructReq(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
+                                                     BatchDownloadReq batchDownloadReq) {
+        DateConf dateConf = batchDownloadReq.getDateInfo();
         Set<Long> modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet());
         modelIds.add(metricResp.getModelId());
-        QueryStructReq queryStructReq = new QueryStructReq();
-        queryStructReq.setGroups(dimensionResps.stream().map(DimSchemaResp::getBizName).collect(Collectors.toList()));
-        queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
+        DownloadStructReq downloadStructReq = new DownloadStructReq();
+        downloadStructReq.setGroups(dimensionResps.stream()
+                .map(DimSchemaResp::getBizName).collect(Collectors.toList()));
+        downloadStructReq.getGroups().add(0, getTimeDimension(dateConf));
         Aggregator aggregator = new Aggregator();
         aggregator.setColumn(metricResp.getBizName());
-        queryStructReq.setAggregators(Lists.newArrayList(aggregator));
-        queryStructReq.setDateInfo(dateConf);
-        queryStructReq.setModelIds(modelIds);
-        queryStructReq.setLimit(10000L);
-        return queryService.queryByStructWithAuth(queryStructReq, user);
+        downloadStructReq.setAggregators(Lists.newArrayList(aggregator));
+        downloadStructReq.setDateInfo(dateConf);
+        downloadStructReq.setModelIds(modelIds);
+        downloadStructReq.setLimit(downloadSize);
+        downloadStructReq.setIsTransform(batchDownloadReq.isTransform());
+        return downloadStructReq;
     }
 
     private String getTimeDimension(DateConf dateConf) {
@@ -257,8 +258,8 @@ public class DownloadServiceImpl implements DownloadService {
                 .collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp));
     }
 
-    private Map<String, String> getDimensionNameMap(List<DimSchemaResp> dimensionResps) {
-        return dimensionResps.stream().collect(Collectors.toMap(DimensionResp::getName, SchemaItem::getBizName));
+    private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
+        return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
     }
 
     private List<DimSchemaResp> getMetricRelaDimensions(MetricSchemaResp metricSchemaResp,
@@ -272,6 +273,32 @@ public class DownloadServiceImpl implements DownloadService {
                 .filter(Objects::nonNull)
                 .collect(Collectors.toList());
     }
+
+    private void downloadFile(HttpServletResponse response, File file, String filename) {
+        try {
+            byte[] buffer = readFileToByteArray(file);
+            response.reset();
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
+            response.addHeader("Content-Length", "" + file.length());
+            try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
+                response.setContentType("application/octet-stream");
+                outputStream.write(buffer);
+                outputStream.flush();
+            }
+        } catch (Exception e) {
+            log.error("failed to download file", e);
+        }
+    }
+
+    private byte[] readFileToByteArray(File file) throws IOException {
+        try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
+            byte[] buffer = new byte[fis.available()];
+            fis.read(buffer);
+            return buffer;
+        }
+    }
+
 }
 
 

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java

@@ -13,9 +13,9 @@ import java.util.stream.Collectors;
 
 public class DataTransformUtils {
 
-    public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
-                                                      List<String> dateList, String metric, List<String> groups,
-                                                      DateConf dateConf) {
+    public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData, String metric,
+                                                      List<String> groups, DateConf dateConf) {
+        List<String> dateList = dateConf.getDateList();
         List<Map<String, Object>> transposedData = new ArrayList<>();
         for (Map<String, Object> originalRow : originalData) {
             Map<String, Object> transposedRow = new HashMap<>();

semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java

@@ -20,10 +20,8 @@ class DataTransformUtilsTest {
         inputData.add(createMap("2023/10/14", "a", "c", "4"));
         inputData.add(createMap("2023/10/15", "b", "b", "5"));
         List<String> groups = Lists.newArrayList("d1", "d2");
-        List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12",
-                "2023/10/13", "2023/10/14", "2023/10/15");
         String metric = "m1";
-        List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList,
+        List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData,
                 metric, groups, new DateConf());
         Assertions.assertEquals(3, resultData.size());
     }