From a9bb1c1f6827236f9430501d366fbd994a6a1281 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Tue, 17 Oct 2023 13:20:20 +0800 Subject: [PATCH] (feature)add metric check parser in chat and add metric check convert in semantic, download metric data in semantic (#241) * (improvement)(chat) add metric check parser * (improvement)(semantic) support metric data download --------- Co-authored-by: jolunoluo --- .../chat/parser/rule/MetricCheckParser.java | 100 ++++++++++++++++++ .../rule/metric/MetricSemanticQuery.java | 53 ---------- .../chat/rest/RecommendController.java | 2 +- .../supersonic/common/pojo/ReturnCode.java | 1 + .../supersonic/common/util/DateUtils.java | 6 ++ .../main/resources/META-INF/spring.factories | 1 + .../advice/RestExceptionHandler.java | 8 ++ .../main/resources/META-INF/spring.factories | 1 + pom.xml | 1 + .../api/model/response/MetricResp.java | 13 +++ semantic/query/pom.xml | 5 + .../parser/convert/MetricCheckConverter.java | 80 ++++++++++++++ .../semantic/query/rest/QueryController.java | 9 ++ .../semantic/query/service/QueryService.java | 3 + .../query/service/QueryServiceImpl.java | 63 +++++++++++ .../query/utils/ComponentFactory.java | 2 + 16 files changed, 294 insertions(+), 54 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/MetricCheckParser.java create mode 100644 semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/MetricCheckParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/MetricCheckParser.java new file mode 100644 index 000000000..fb9ffbd66 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/MetricCheckParser.java @@ -0,0 +1,100 @@ +package com.tencent.supersonic.chat.parser.rule; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.chat.api.component.SemanticInterpreter; +import com.tencent.supersonic.chat.api.component.SemanticParser; +import com.tencent.supersonic.chat.api.component.SemanticQuery; +import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; +import com.tencent.supersonic.chat.utils.ComponentFactory; +import org.apache.commons.collections.CollectionUtils; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class MetricCheckParser implements SemanticParser { + + @Override + public void parse(QueryContext queryContext, ChatContext chatContext) { + List semanticQueries = queryContext.getCandidateQueries(); + if (CollectionUtils.isEmpty(semanticQueries)) { + return; + } + semanticQueries.removeIf(this::removeQuery); + } + + private boolean removeQuery(SemanticQuery semanticQuery) { + if (semanticQuery instanceof MetricSemanticQuery) { + SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); + List schemaElementMatches = parseInfo.getElementMatches(); + List elementMatchFiltered = + filterMetricElement(schemaElementMatches, parseInfo.getModelId()); + return 0 >= getMetricElementMatchCount(elementMatchFiltered); + } + return false; + } + + private List filterMetricElement(List elementMatches, Long modelId) { + List filterSchemaElementMatch = Lists.newArrayList(); + SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); + ModelSchema modelSchema = semanticInterpreter.getModelSchema(modelId, true); + Set metricElements = modelSchema.getMetrics(); + Map valueElementMatchMap = getValueElementMap(elementMatches); + Map metricMap = metricElements.stream() + .collect(Collectors.toMap(SchemaElement::getId, e -> e, (e1, e2) -> e2)); + for (SchemaElementMatch schemaElementMatch : elementMatches) { + if (!SchemaElementType.METRIC.equals(schemaElementMatch.getElement().getType())) { + filterSchemaElementMatch.add(schemaElementMatch); + continue; + } + SchemaElement metric = metricMap.get(schemaElementMatch.getElement().getId()); + List necessaryDimensionIds = getNecessaryDimensionIds(metric); + boolean flag = true; + for (Long necessaryDimensionId : necessaryDimensionIds) { + if (!valueElementMatchMap.containsKey(necessaryDimensionId)) { + flag = false; + break; + } + } + if (flag) { + filterSchemaElementMatch.add(schemaElementMatch); + } + } + return filterSchemaElementMatch; + } + + private Map getValueElementMap(List elementMatches) { + return elementMatches.stream() + .filter(elementMatch -> + SchemaElementType.VALUE.equals(elementMatch.getElement().getType())) + .collect(Collectors.toMap(elementMatch -> elementMatch.getElement().getId(), e -> e, (e1, e2) -> e1)); + } + + private long getMetricElementMatchCount(List elementMatches) { + return elementMatches.stream().filter(elementMatch -> + SchemaElementType.METRIC.equals(elementMatch.getElement().getType())) + .count(); + } + + private List getNecessaryDimensionIds(SchemaElement metric) { + if (metric == null) { + return Lists.newArrayList(); + } + List relateSchemaElements = metric.getRelateSchemaElements(); + if (CollectionUtils.isEmpty(relateSchemaElements)) { + return Lists.newArrayList(); + } + return relateSchemaElements.stream() + .filter(RelateSchemaElement::isNecessary).map(RelateSchemaElement::getDimensionId) + .collect(Collectors.toList()); + } + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java index 45af482ba..61f66bf3f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java @@ -3,14 +3,9 @@ package com.tencent.supersonic.chat.query.rule.metric; import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.ModelSchema; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; @@ -28,11 +23,7 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR import java.time.LocalDate; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -86,53 +77,9 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { filteredMatches.add(schemaElementMatch); } } - filteredMatches = metricRelateDimensionCheck(filteredMatches, modelId); return filteredMatches; } - private List metricRelateDimensionCheck(List elementMatches, Long modelId) { - List filterSchemaElementMatch = Lists.newArrayList(); - - ModelSchema modelSchema = semanticInterpreter.getModelSchema(modelId, true); - Set metricElements = modelSchema.getMetrics(); - Map valueElementMatchMap = elementMatches.stream() - .filter(elementMatch -> - SchemaElementType.VALUE.equals(elementMatch.getElement().getType()) - || SchemaElementType.ID.equals(elementMatch.getElement().getType())) - .collect(Collectors.toMap(elementMatch -> elementMatch.getElement().getId(), e -> e, (e1, e2) -> e1)); - Map metricMap = metricElements.stream() - .collect(Collectors.toMap(SchemaElement::getId, e -> e, (e1, e2) -> e2)); - - for (SchemaElementMatch schemaElementMatch : elementMatches) { - if (!SchemaElementType.METRIC.equals(schemaElementMatch.getElement().getType())) { - filterSchemaElementMatch.add(schemaElementMatch); - continue; - } - SchemaElement metric = metricMap.get(schemaElementMatch.getElement().getId()); - if (metric == null) { - continue; - } - List relateSchemaElements = metric.getRelateSchemaElements(); - if (CollectionUtils.isEmpty(relateSchemaElements)) { - filterSchemaElementMatch.add(schemaElementMatch); - continue; - } - List necessaryDimensionIds = relateSchemaElements.stream() - .filter(RelateSchemaElement::isNecessary).map(RelateSchemaElement::getDimensionId) - .collect(Collectors.toList()); - boolean flag = true; - for (Long necessaryDimensionId : necessaryDimensionIds) { - if (!valueElementMatchMap.containsKey(necessaryDimensionId)) { - flag = false; - break; - } - } - if (flag) { - filterSchemaElementMatch.add(schemaElementMatch); - } - } - return filterSchemaElementMatch; - } @Override public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java index 68649e5e7..8592dca2c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java @@ -32,7 +32,7 @@ public class RecommendController { @GetMapping("recommend/metric/{modelId}") public RecommendResp recommendMetricMode(@PathVariable("modelId") Long modelId, - @RequestParam(value = "metric", required = false) Long metricId, + @RequestParam(value = "metricId", required = false) Long metricId, @RequestParam(value = "limit", required = false) Long limit) { RecommendReq recommendReq = new RecommendReq(); recommendReq.setModelId(modelId); diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ReturnCode.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ReturnCode.java index fa579a227..e9f806d0e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ReturnCode.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ReturnCode.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.common.pojo; public enum ReturnCode { SUCCESS(200, "success"), + INVALID_REQUEST(400, "invalid request"), INVALID_PERMISSION(401, "invalid permission"), ACCESS_ERROR(403, "access denied"), SYSTEM_ERROR(500, "system error"); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index ea6363c65..82f8c73cf 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -18,6 +18,7 @@ public class DateUtils { public static final String DATE_FIELD = "数据日期"; public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + public static final String FORMAT = "yyyyMMddHHmmss"; public static Integer currentYear() { Date date = new Date(); @@ -116,6 +117,11 @@ public class DateUtils { return dateFormat.format(date); } + public static String format(Date date, String format) { + DateFormat dateFormat = new SimpleDateFormat(format); + return dateFormat.format(date); + } + private static boolean containsTime(Date date) { DateFormat timeFormat = new SimpleDateFormat("HH:mm:ss"); String timeString = timeFormat.format(date); diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 5938c14b0..8117e1a81 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -8,6 +8,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.QueryModeParser, \ com.tencent.supersonic.chat.parser.rule.ContextInheritParser, \ com.tencent.supersonic.chat.parser.rule.AgentCheckParser, \ + com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ diff --git a/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java b/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java index 5b62f20ee..d4e4e6324 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/advice/RestExceptionHandler.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.advice; import com.tencent.supersonic.common.pojo.exception.AccessException; import com.tencent.supersonic.common.pojo.exception.CommonException; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.common.pojo.ResultData; import com.tencent.supersonic.common.pojo.ReturnCode; @@ -39,6 +40,13 @@ public class RestExceptionHandler { return ResultData.fail(ReturnCode.INVALID_PERMISSION.getCode(), e.getMessage()); } + @ExceptionHandler(InvalidArgumentException.class) + @ResponseStatus(HttpStatus.OK) + public ResultData invalidArgumentException(Exception e) { + log.error("default global exception", e); + return ResultData.fail(ReturnCode.INVALID_REQUEST.getCode(), e.getMessage()); + } + @ExceptionHandler(CommonException.class) @ResponseStatus(HttpStatus.OK) public ResultData commonException(CommonException e) { diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 8c590cd4e..54ce3be22 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -8,6 +8,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.QueryModeParser, \ com.tencent.supersonic.chat.parser.rule.ContextInheritParser, \ com.tencent.supersonic.chat.parser.rule.AgentCheckParser, \ + com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ diff --git a/pom.xml b/pom.xml index 2b605c8f7..d8386dfe5 100644 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,7 @@ 22.3.0 + 3.1.1 diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java index a3f9b8e44..7ab64d94a 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MetricResp.java @@ -2,15 +2,20 @@ package com.tencent.supersonic.semantic.api.model.response; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.DataFormat; +import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import lombok.Data; import lombok.ToString; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; @Data @@ -49,4 +54,12 @@ public class MetricResp extends SchemaItem { tags = Arrays.asList(tag.split(",")); } } + + public Set getNecessaryDimensionIds() { + if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { + return Sets.newHashSet(); + } + return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary) + .map(DrillDownDimension::getDimensionId).collect(Collectors.toSet()); + } } diff --git a/semantic/query/pom.xml b/semantic/query/pom.xml index a92aef702..481dd4f01 100644 --- a/semantic/query/pom.xml +++ b/semantic/query/pom.xml @@ -92,6 +92,11 @@ + + com.alibaba + easyexcel + ${easyexcel.version} + diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java new file mode 100644 index 000000000..49b2f6451 --- /dev/null +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java @@ -0,0 +1,80 @@ +package com.tencent.supersonic.semantic.query.parser.convert; + +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; +import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.query.pojo.Filter; +import com.tencent.supersonic.semantic.api.query.request.MetricReq; +import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import com.tencent.supersonic.semantic.model.domain.Catalog; +import com.tencent.supersonic.semantic.query.parser.SemanticConverter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +@Component("MetricCheckConverter") +@Slf4j +public class MetricCheckConverter implements SemanticConverter { + + @Override + public boolean accept(QueryStructReq queryStructCmd) { + if (queryStructCmd.getNativeQuery()) { + return false; + } + return !CollectionUtils.isEmpty(queryStructCmd.getAggregators()); + } + + @Override + public void converter(Catalog catalog, QueryStructReq queryStructReq, ParseSqlReq sqlCommend, + MetricReq metricCommand) throws Exception { + Long modelId = queryStructReq.getModelId(); + List metricResps = catalog.getMetrics(modelId); + List dimensionResps = catalog.getDimensions(modelId); + Map dimensionMap = dimensionResps.stream() + .collect(Collectors.toMap(DimensionResp::getId, d -> d)); + List metricBizNames = queryStructReq.getMetrics(); + List dimensionFilterBizNames = queryStructReq.getDimensionFilters().stream() + .map(Filter::getBizName).collect(Collectors.toList()); + List metricToQuery = metricResps.stream().filter(metricResp -> + metricBizNames.contains(metricResp.getBizName())).collect(Collectors.toList()); + List dimensionToFilter = dimensionResps.stream().filter(dimensionResp -> + dimensionFilterBizNames.contains(dimensionResp.getBizName())) + .map(DimensionResp::getId).collect(Collectors.toList()); + for (MetricResp metricResp : metricToQuery) { + Set necessaryDimensionIds = metricResp.getNecessaryDimensionIds(); + if (CollectionUtils.isEmpty(necessaryDimensionIds)) { + continue; + } + DimensionResp dimensionResp = null; + for (Long dimensionId : necessaryDimensionIds) { + dimensionResp = dimensionMap.get(dimensionId); + if (dimensionResp != null) { + break; + } + } + if (dimensionResp == null) { + continue; + } + String message = String.format("该指标必须配合维度[%s]来进行过滤查询", dimensionResp.getName()); + if (CollectionUtils.isEmpty(dimensionToFilter)) { + throw new InvalidArgumentException(message); + } + boolean flag = false; + for (Long dimensionId : dimensionToFilter) { + if (necessaryDimensionIds.contains(dimensionId)) { + flag = true; + break; + } + } + if (!flag) { + throw new InvalidArgumentException(message); + } + } + } +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java index 203199da6..97564dd7a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java @@ -59,6 +59,15 @@ public class QueryController { return queryService.queryByStructWithAuth(queryStructReq, user); } + @PostMapping("/download/struct") + public void downloadByStruct(@RequestBody QueryStructReq queryStructReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + queryService.downloadByStruct(queryStructReq, user, response); + + } + @PostMapping("/queryStatement") public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception { return queryService.queryByQueryStatement(queryStatement); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java index b7221ff04..8685d8652 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import javax.servlet.http.HttpServletResponse; import java.util.List; public interface QueryService { @@ -21,6 +22,8 @@ public interface QueryService { QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; + void downloadByStruct(QueryStructReq queryStructReq, User user, HttpServletResponse response) throws Exception; + QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index f9308fb68..ac8525bac 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -1,10 +1,14 @@ package com.tencent.supersonic.semantic.query.service; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.util.FileUtils; import com.google.common.cache.CacheBuilder; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.common.util.ContextUtils; @@ -30,16 +34,28 @@ import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import com.tencent.supersonic.semantic.query.utils.StatUtils; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URLEncoder; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.assertj.core.util.Lists; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import javax.servlet.http.HttpServletResponse; @Service @@ -146,6 +162,53 @@ public class QueryServiceImpl implements QueryService { } } + @Override + public void downloadByStruct(QueryStructReq queryStructReq, + User user, HttpServletResponse response) throws Exception { + QueryResultWithSchemaResp queryResultWithSchemaResp = queryByStruct(queryStructReq, user); + List> data = new ArrayList<>(); + List> header = Lists.newArrayList(); + for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + header.add(Lists.newArrayList(column.getName())); + } + for (Map row : queryResultWithSchemaResp.getResultList()) { + List rowData = new ArrayList<>(); + for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + rowData.add(String.valueOf(row.get(column.getNameEn()))); + } + data.add(rowData); + } + String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); + File file = FileUtils.createTmpFile(fileName); + EasyExcel.write(file).sheet("Sheet1").head(header).doWrite(data); + downloadFile(response, file, fileName); + } + + private void downloadFile(HttpServletResponse response, File file, String filename) { + try { + byte[] buffer = readFileToByteArray(file); + response.reset(); + response.setCharacterEncoding("UTF-8"); + response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8")); + response.addHeader("Content-Length", "" + file.length()); + try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) { + response.setContentType("application/octet-stream"); + outputStream.write(buffer); + outputStream.flush(); + } + } catch (Exception e) { + log.error("failed to download file", e); + } + } + + private byte[] readFileToByteArray(File file) throws IOException { + try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) { + byte[] buffer = new byte[fis.available()]; + fis.read(buffer); + return buffer; + } + } + @Override @DataPermission @SneakyThrows diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java index 96fd31cac..c0dd1f033 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.query.parser.SqlParser; import com.tencent.supersonic.semantic.query.parser.calcite.CalciteSqlParser; import com.tencent.supersonic.semantic.query.parser.convert.CalculateAggConverter; import com.tencent.supersonic.semantic.query.parser.convert.DefaultDimValueConverter; +import com.tencent.supersonic.semantic.query.parser.convert.MetricCheckConverter; import com.tencent.supersonic.semantic.query.parser.convert.MultiSourceJoin; import com.tencent.supersonic.semantic.query.parser.convert.ParserDefaultConverter; import java.util.ArrayList; @@ -78,6 +79,7 @@ public class ComponentFactory { } private static void initSemanticConverter() { + semanticConverters.add(getBean("MetricCheckConverter", MetricCheckConverter.class)); semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));