mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 03:58:14 +00:00
(improvement)(Headless) Metric Market only displays published metrics (#867)
* (improvement)(Headless) Remove Query controller * (improvement)(Headless) Filter metrics and dimensions based on whether they were created as tags * (improvement)(Headless) Metric Market only displays published metrics --------- Co-authored-by: jolunoluo
This commit is contained in:
@@ -25,6 +25,7 @@ public class NL2SQLParser implements ChatParser {
|
||||
if (!ParseResp.ParseState.FAILED.equals(text2SqlParseResp.getState())) {
|
||||
parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
|
||||
}
|
||||
parseResp.getParseTimeCost().setSqlTime(text2SqlParseResp.getParseTimeCost().getSqlTime());
|
||||
}
|
||||
|
||||
private boolean checkSkip(ParseResp parseResp) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.tencent.supersonic.common.pojo;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import lombok.Data;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -46,6 +47,8 @@ public class DateConf {
|
||||
|
||||
private boolean isInherited;
|
||||
|
||||
private boolean groupByDate;
|
||||
|
||||
public List<String> getDateList() {
|
||||
if (!CollectionUtils.isEmpty(dateList)) {
|
||||
return dateList;
|
||||
@@ -55,6 +58,18 @@ public class DateConf {
|
||||
return DateUtils.getDateList(startDateStr, endDateStr, getPeriod());
|
||||
}
|
||||
|
||||
public String getGroupByTimeDimension() {
|
||||
if (Constants.DAY.equals(period)) {
|
||||
return TimeDimensionEnum.DAY.getName();
|
||||
} else if (Constants.WEEK.equals(period)) {
|
||||
return TimeDimensionEnum.WEEK.getName();
|
||||
} else if (Constants.MONTH.equals(period)) {
|
||||
return TimeDimensionEnum.MONTH.getName();
|
||||
} else {
|
||||
return TimeDimensionEnum.DAY.getName();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
|
||||
@@ -2,10 +2,10 @@ package com.tencent.supersonic.common.pojo;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import java.util.List;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@@ -13,6 +13,7 @@ import lombok.NoArgsConstructor;
|
||||
public class Filter {
|
||||
|
||||
private Relation relation = Relation.FILTER;
|
||||
private Long id;
|
||||
private String bizName;
|
||||
private String name;
|
||||
private FilterOperatorEnum operator;
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class DownloadStructReq extends QueryStructReq {
|
||||
public class DownloadMetricReq extends QueryMetricReq {
|
||||
|
||||
private boolean isTransform;
|
||||
|
||||
@@ -11,4 +11,6 @@ public class PageMetricReq extends PageSchemaItemReq {
|
||||
|
||||
private Integer isTag;
|
||||
|
||||
private Integer isPublish;
|
||||
|
||||
}
|
||||
|
||||
@@ -1,23 +1,28 @@
|
||||
package com.tencent.supersonic.headless.api.pojo.request;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import java.util.List;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class QueryMetricReq {
|
||||
|
||||
private Long domainId;
|
||||
|
||||
private List<Long> metricIds;
|
||||
private List<Long> metricIds = Lists.newArrayList();
|
||||
|
||||
private List<String> metricNames;
|
||||
private List<String> metricNames = Lists.newArrayList();
|
||||
|
||||
private List<Long> dimensionIds;
|
||||
private List<Long> dimensionIds = Lists.newArrayList();
|
||||
|
||||
private List<String> dimensionNames;
|
||||
private List<String> dimensionNames = Lists.newArrayList();
|
||||
|
||||
private List<Filter> filters = Lists.newArrayList();
|
||||
|
||||
private DateConf dateInfo = new DateConf();
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ public class MetricResp extends SchemaItem {
|
||||
|
||||
private int isTag;
|
||||
|
||||
private Integer isPublish;
|
||||
|
||||
public void setClassifications(String tag) {
|
||||
if (StringUtils.isBlank(tag)) {
|
||||
classifications = Lists.newArrayList();
|
||||
|
||||
@@ -103,4 +103,6 @@ public class MetricDO {
|
||||
|
||||
private String defineType;
|
||||
|
||||
private Integer isPublish;
|
||||
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.server.persistence.mapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import java.util.List;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface MetricDOCustomMapper {
|
||||
@@ -13,6 +13,10 @@ public interface MetricDOCustomMapper {
|
||||
|
||||
void batchUpdateStatus(List<MetricDO> metricDOS);
|
||||
|
||||
void batchPublish(List<MetricDO> metricDOS);
|
||||
|
||||
void batchUnPublish(List<MetricDO> metricDOS);
|
||||
|
||||
List<MetricDO> query(MetricFilter metricFilter);
|
||||
|
||||
List<MetricDO> queryMetrics(MetricsFilter metricsFilter);
|
||||
|
||||
@@ -18,6 +18,10 @@ public interface MetricRepository {
|
||||
|
||||
void batchUpdateStatus(List<MetricDO> metricDOS);
|
||||
|
||||
void batchPublish(List<MetricDO> metricDOS);
|
||||
|
||||
void batchUnPublish(List<MetricDO> metricDOS);
|
||||
|
||||
MetricDO getMetricById(Long id);
|
||||
|
||||
List<MetricDO> getMetric(MetricFilter metricFilter);
|
||||
|
||||
@@ -52,6 +52,16 @@ public class MetricRepositoryImpl implements MetricRepository {
|
||||
metricDOCustomMapper.batchUpdateStatus(metricDOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchPublish(List<MetricDO> metricDOS) {
|
||||
metricDOCustomMapper.batchPublish(metricDOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchUnPublish(List<MetricDO> metricDOS) {
|
||||
metricDOCustomMapper.batchUnPublish(metricDOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MetricDO getMetricById(Long id) {
|
||||
return metricDOMapper.selectById(id);
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.pojo;
|
||||
import com.google.common.base.Objects;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@@ -26,7 +27,7 @@ public class MetaFilter {
|
||||
|
||||
private Integer sensitiveLevel;
|
||||
|
||||
private Integer status;
|
||||
private List<Integer> status;
|
||||
|
||||
private String key;
|
||||
|
||||
|
||||
@@ -7,4 +7,7 @@ public class MetricFilter extends MetaFilter {
|
||||
|
||||
private String type;
|
||||
|
||||
private String userName;
|
||||
|
||||
private Integer isPublish;
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ public class SqlInfoProcessor implements ResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
||||
long start = System.currentTimeMillis();
|
||||
List<SemanticQuery> semanticQueries = queryContext.getCandidateQueries();
|
||||
if (CollectionUtils.isEmpty(semanticQueries)) {
|
||||
return;
|
||||
@@ -41,6 +42,7 @@ public class SqlInfoProcessor implements ResultProcessor {
|
||||
List<SemanticParseInfo> selectedParses = semanticQueries.stream().map(SemanticQuery::getParseInfo)
|
||||
.collect(Collectors.toList());
|
||||
addSqlInfo(queryContext, selectedParses);
|
||||
parseResp.getParseTimeCost().setSqlTime(System.currentTimeMillis() - start);
|
||||
}
|
||||
|
||||
private void addSqlInfo(QueryContext queryContext, List<SemanticParseInfo> semanticParseInfos) {
|
||||
|
||||
@@ -9,10 +9,14 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
@@ -30,11 +34,11 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/dimension")
|
||||
public class DimensionController {
|
||||
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
public DimensionController(DimensionService dimensionService) {
|
||||
this.dimensionService = dimensionService;
|
||||
}
|
||||
@Autowired
|
||||
private QueryService queryService;
|
||||
|
||||
|
||||
/**
|
||||
@@ -107,6 +111,14 @@ public class DimensionController {
|
||||
return dimensionService.queryDimension(pageDimensionReq);
|
||||
}
|
||||
|
||||
@PostMapping("/queryDimValue")
|
||||
public SemanticQueryResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryDimValue(queryDimValueReq, user);
|
||||
}
|
||||
|
||||
@DeleteMapping("deleteDimension/{id}")
|
||||
public Boolean deleteDimension(@PathVariable("id") Long id,
|
||||
HttpServletRequest request,
|
||||
|
||||
@@ -65,6 +65,24 @@ public class MetricController {
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchPublish")
|
||||
public Boolean batchPublish(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchPublish(metaBatchReq.getIds(), user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchUnPublish")
|
||||
public Boolean batchUnPublish(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchUnPublish(metaBatchReq.getIds(), user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/mockMetricAlias")
|
||||
public List<String> mockMetricAlias(@RequestBody MetricBaseReq metricReq,
|
||||
HttpServletRequest request,
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
package com.tencent.supersonic.headless.server.rest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.server.service.DownloadService;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.validation.Valid;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/semantic/query")
|
||||
@Slf4j
|
||||
public class QueryController {
|
||||
|
||||
@Autowired
|
||||
private QueryService queryService;
|
||||
|
||||
@Autowired
|
||||
private DownloadService downloadService;
|
||||
|
||||
@PostMapping("/struct")
|
||||
public Object queryByStruct(@RequestBody QueryStructReq queryStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert(true);
|
||||
return queryService.queryByReq(querySqlReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/queryMetricDataById")
|
||||
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
|
||||
HttpServletRequest request) throws Exception {
|
||||
return queryService.queryMetricDataById(queryApiReq, request);
|
||||
}
|
||||
|
||||
@PostMapping("/download/struct")
|
||||
public void downloadByStruct(@RequestBody DownloadStructReq downloadStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.downloadByStruct(downloadStructReq, user, response);
|
||||
}
|
||||
|
||||
@PostMapping("/download/batch")
|
||||
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.batchDownload(batchDownloadReq, user, response);
|
||||
}
|
||||
|
||||
/**
|
||||
* queryByMultiStruct
|
||||
*/
|
||||
@PostMapping("/multiStruct")
|
||||
public Object queryByMultiStruct(@RequestBody QueryMultiStructReq queryMultiStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryByReq(queryMultiStructReq, user);
|
||||
}
|
||||
|
||||
/**
|
||||
* getStatInfo
|
||||
* query the used frequency of the metric/dimension
|
||||
*
|
||||
* @param itemUseReq
|
||||
*/
|
||||
@PostMapping("/stat")
|
||||
public List<ItemUseResp> getStatInfo(@RequestBody ItemUseReq itemUseReq) {
|
||||
return queryService.getStatInfo(itemUseReq);
|
||||
}
|
||||
|
||||
@PostMapping("/queryDimValue")
|
||||
public SemanticQueryResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryDimValue(queryDimValueReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/explain")
|
||||
public <T> ExplainResp explain(@RequestBody ExplainSqlReq<T> explainSqlReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq());
|
||||
|
||||
if (QueryMethod.SQL.equals(explainSqlReq.getQueryTypeEnum())) {
|
||||
ExplainSqlReq<QuerySqlReq> explainSqlReqNew = ExplainSqlReq.<QuerySqlReq>builder()
|
||||
.queryReq(JsonUtil.toObject(queryReqJson, QuerySqlReq.class))
|
||||
.queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build();
|
||||
return queryService.explain(explainSqlReqNew, user);
|
||||
}
|
||||
if (QueryMethod.STRUCT.equals(explainSqlReq.getQueryTypeEnum())) {
|
||||
ExplainSqlReq<QueryStructReq> explainSqlReqNew = ExplainSqlReq.<QueryStructReq>builder()
|
||||
.queryReq(JsonUtil.toObject(queryReqJson, QueryStructReq.class))
|
||||
.queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build();
|
||||
return queryService.explain(explainSqlReqNew, user);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package com.tencent.supersonic.headless.server.rest.api;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.server.service.DownloadService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -27,13 +30,32 @@ public class MetricQueryApiController {
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Autowired
|
||||
private DownloadService downloadService;
|
||||
|
||||
@PostMapping("/metric")
|
||||
public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
|
||||
return queryService.queryByReq(queryStructReq.convert(), user);
|
||||
return queryService.queryByReq(queryStructReq.convert(true), user);
|
||||
}
|
||||
|
||||
@PostMapping("/download/metric")
|
||||
public void downloadMetric(@RequestBody DownloadMetricReq downloadMetricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.downloadByStruct(downloadMetricReq, user, response);
|
||||
}
|
||||
|
||||
@PostMapping("/downloadBatch/metric")
|
||||
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.batchDownload(batchDownloadReq, user, response);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@ package com.tencent.supersonic.headless.server.service;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
public interface DownloadService {
|
||||
|
||||
void downloadByStruct(DownloadStructReq downloadStructReq,
|
||||
void downloadByStruct(DownloadMetricReq downloadStructReq,
|
||||
User user, HttpServletResponse response) throws Exception;
|
||||
|
||||
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;
|
||||
|
||||
@@ -27,6 +27,10 @@ public interface MetricService {
|
||||
|
||||
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
|
||||
|
||||
void batchPublish(List<Long> metricIds, User user);
|
||||
|
||||
void batchUnPublish(List<Long> metricIds, User user);
|
||||
|
||||
void deleteMetric(Long id, User user) throws Exception;
|
||||
|
||||
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
|
||||
|
||||
@@ -4,15 +4,11 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.List;
|
||||
|
||||
public interface QueryService {
|
||||
@@ -25,6 +21,4 @@ public interface QueryService {
|
||||
|
||||
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
|
||||
|
||||
@ApiHeaderCheck
|
||||
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,13 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.DomainService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
@@ -43,13 +50,6 @@ import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Service
|
||||
public class DataSetServiceImpl
|
||||
@@ -109,8 +109,8 @@ public class DataSetServiceImpl
|
||||
if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
|
||||
wrapper.lambda().in(DataSetDO::getId, metaFilter.getIds());
|
||||
}
|
||||
if (metaFilter.getStatus() != null) {
|
||||
wrapper.lambda().eq(DataSetDO::getStatus, metaFilter.getStatus());
|
||||
if (!CollectionUtils.isEmpty(metaFilter.getStatus())) {
|
||||
wrapper.lambda().in(DataSetDO::getStatus, metaFilter.getStatus());
|
||||
}
|
||||
wrapper.lambda().ne(DataSetDO::getStatus, StatusEnum.DELETED.getCode());
|
||||
return list(wrapper).stream().map(entry -> convert(entry, user)).collect(Collectors.toList());
|
||||
@@ -212,7 +212,7 @@ public class DataSetServiceImpl
|
||||
@Override
|
||||
public Map<Long, List<Long>> getModelIdToDataSetIds(List<Long> dataSetIds, User user) {
|
||||
MetaFilter metaFilter = new MetaFilter();
|
||||
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
|
||||
metaFilter.setStatus(Lists.newArrayList(StatusEnum.ONLINE.getCode()));
|
||||
metaFilter.setIds(dataSetIds);
|
||||
List<DataSetResp> dataSetList = dataSetSchemaCache.getIfPresent(metaFilter);
|
||||
if (CollectionUtils.isEmpty(dataSetList)) {
|
||||
|
||||
@@ -16,8 +16,9 @@ import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
@@ -73,14 +74,14 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void downloadByStruct(DownloadStructReq downloadStructReq,
|
||||
User user, HttpServletResponse response) throws Exception {
|
||||
public void downloadByStruct(DownloadMetricReq downloadMetricReq,
|
||||
User user, HttpServletResponse response) throws Exception {
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
try {
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert(true);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
QueryStructReq queryStructReq = metricService.convert(downloadMetricReq);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(queryStructReq.convert(true), user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, queryStructReq, downloadMetricReq.isTransform());
|
||||
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
|
||||
} catch (RuntimeException e) {
|
||||
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
|
||||
@@ -126,11 +127,12 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
List<DimensionResp> dimensions = getMetricRelaDimensions(metricResp, dimensionRespMap);
|
||||
for (MetricResp metric : metrics) {
|
||||
try {
|
||||
DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq);
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert();
|
||||
QueryStructReq queryStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq);
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert();
|
||||
querySqlReq.setNeedAuth(true);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult,
|
||||
queryStructReq, batchDownloadReq.isTransform());
|
||||
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
|
||||
.head(dataDownload.getHeaders()).build();
|
||||
excelWriter.write(dataDownload.getData(), writeSheet);
|
||||
@@ -215,15 +217,16 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
return data;
|
||||
}
|
||||
|
||||
private DataDownload buildDataDownload(SemanticQueryResp queryResult, DownloadStructReq downloadStructReq) {
|
||||
private DataDownload buildDataDownload(SemanticQueryResp queryResult,
|
||||
QueryStructReq queryStructReq, boolean isTransform) {
|
||||
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
|
||||
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
|
||||
if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {
|
||||
if (isTransform && !CollectionUtils.isEmpty(metricColumns)) {
|
||||
QueryColumn metric = metricColumns.get(0);
|
||||
List<String> groups = downloadStructReq.getGroups();
|
||||
List<String> groups = queryStructReq.getGroups();
|
||||
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(),
|
||||
metric.getNameEn(), groups, downloadStructReq.getDateInfo());
|
||||
List<List<String>> headers = buildHeader(dimensionColumns, downloadStructReq.getDateInfo().getDateList());
|
||||
metric.getNameEn(), groups, queryStructReq.getDateInfo());
|
||||
List<List<String>> headers = buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList());
|
||||
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
|
||||
dataTransformed, metric.getName());
|
||||
return DataDownload.builder().headers(headers).data(data).build();
|
||||
@@ -234,23 +237,22 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
}
|
||||
|
||||
private DownloadStructReq buildDownloadReq(List<DimensionResp> dimensionResps, MetricResp metricResp,
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
private QueryStructReq buildDownloadReq(List<DimensionResp> dimensionResps, MetricResp metricResp,
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
DateConf dateConf = batchDownloadReq.getDateInfo();
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
DownloadStructReq downloadStructReq = new DownloadStructReq();
|
||||
downloadStructReq.setGroups(dimensionResps.stream()
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName).collect(Collectors.toList()));
|
||||
downloadStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
downloadStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
downloadStructReq.setDateInfo(dateConf);
|
||||
downloadStructReq.setModelIds(modelIds);
|
||||
downloadStructReq.setLimit(downloadSize);
|
||||
downloadStructReq.setIsTransform(batchDownloadReq.isTransform());
|
||||
return downloadStructReq;
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setModelIds(modelIds);
|
||||
queryStructReq.setLimit(downloadSize);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataItem;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
@@ -56,6 +57,7 @@ import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -190,6 +192,30 @@ public class MetricServiceImpl implements MetricService {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchPublish(List<Long> metricIds, User user) {
|
||||
MetricFilter metricFilter = new MetricFilter();
|
||||
metricFilter.setIds(metricIds);
|
||||
List<MetricDO> metrics = metricRepository.getMetric(metricFilter);
|
||||
for (MetricDO metricDO : metrics) {
|
||||
metricDO.setUpdatedAt(new Date());
|
||||
metricDO.setUpdatedBy(user.getName());
|
||||
}
|
||||
metricRepository.batchPublish(metrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchUnPublish(List<Long> metricIds, User user) {
|
||||
MetricFilter metricFilter = new MetricFilter();
|
||||
metricFilter.setIds(metricIds);
|
||||
List<MetricDO> metrics = metricRepository.getMetric(metricFilter);
|
||||
for (MetricDO metricDO : metrics) {
|
||||
metricDO.setUpdatedAt(new Date());
|
||||
metricDO.setUpdatedBy(user.getName());
|
||||
}
|
||||
metricRepository.batchUnPublish(metrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteMetric(Long id, User user) {
|
||||
MetricDO metricDO = metricRepository.getMetricById(id);
|
||||
@@ -206,6 +232,7 @@ public class MetricServiceImpl implements MetricService {
|
||||
@Override
|
||||
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
|
||||
MetricFilter metricFilter = new MetricFilter();
|
||||
metricFilter.setUserName(user.getName());
|
||||
BeanUtils.copyProperties(pageMetricReq, metricFilter);
|
||||
List<ModelResp> modelResps = modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds());
|
||||
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
|
||||
@@ -550,7 +577,9 @@ public class MetricServiceImpl implements MetricService {
|
||||
//2. get metrics and dimensions
|
||||
List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
|
||||
|
||||
List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
|
||||
List<DimensionResp> dimensionResps = getDimensionResps(modelIdsByDomainId);
|
||||
Map<Long, DimensionResp> dimensionRespMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
|
||||
|
||||
//3. choose ModelCluster
|
||||
Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
|
||||
@@ -559,16 +588,22 @@ public class MetricServiceImpl implements MetricService {
|
||||
//4. set groups
|
||||
List<String> dimensionBizNames = dimensionResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(entry -> entry.getBizName()).collect(Collectors.toList());
|
||||
.filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
|
||||
|| queryMetricReq.getDimensionNames().contains(entry.getBizName())
|
||||
|| queryMetricReq.getDimensionIds().contains(entry.getId()))
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(dimensionBizNames)) {
|
||||
queryStructReq.setGroups(dimensionBizNames);
|
||||
if (queryMetricReq.getDateInfo().isGroupByDate()) {
|
||||
queryStructReq.getGroups().add(queryMetricReq.getDateInfo().getGroupByTimeDimension());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimensionBizNames)) {
|
||||
queryStructReq.getGroups().addAll(dimensionBizNames);
|
||||
}
|
||||
//5. set aggregators
|
||||
List<String> metricBizNames = metricResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(entry -> entry.getBizName()).collect(Collectors.toList());
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
if (org.apache.commons.collections.CollectionUtils.isEmpty(metricBizNames)) {
|
||||
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
|
||||
}
|
||||
@@ -584,6 +619,16 @@ public class MetricServiceImpl implements MetricService {
|
||||
for (Long modelId : modelCluster.getModelIds()) {
|
||||
queryStructReq.addModelId(modelId);
|
||||
}
|
||||
List<Filter> filters = queryMetricReq.getFilters();
|
||||
for (Filter filter : filters) {
|
||||
if (StringUtils.isBlank(filter.getBizName())) {
|
||||
DimensionResp dimensionResp = dimensionRespMap.get(filter.getId());
|
||||
if (dimensionResp != null) {
|
||||
filter.setBizName(dimensionResp.getBizName());
|
||||
}
|
||||
}
|
||||
}
|
||||
queryStructReq.setDimensionFilters(filters);
|
||||
//7. set dateInfo
|
||||
queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
|
||||
return queryStructReq;
|
||||
@@ -626,9 +671,8 @@ public class MetricServiceImpl implements MetricService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
|
||||
private List<DimensionResp> getDimensionResps(Set<Long> modelIds) {
|
||||
DimensionsFilter dimensionsFilter = new DimensionsFilter();
|
||||
BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
|
||||
dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
|
||||
return dimensionService.queryDimensions(dimensionsFilter);
|
||||
}
|
||||
|
||||
@@ -3,32 +3,21 @@ package com.tencent.supersonic.headless.server.service.impl;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.enums.ApiItemType;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||
import com.tencent.supersonic.headless.api.pojo.Item;
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
@@ -41,27 +30,22 @@ import com.tencent.supersonic.headless.core.planner.QueryPlanner;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
|
||||
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
|
||||
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.service.AppService;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -158,7 +142,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
return null;
|
||||
}
|
||||
|
||||
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) throws Exception {
|
||||
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) {
|
||||
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
|
||||
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
@@ -217,84 +201,6 @@ public class QueryServiceImpl implements QueryService {
|
||||
return getExplainResp(queryStatement);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ItemQueryResultResp queryMetricDataById(QueryItemReq queryItemReq,
|
||||
HttpServletRequest request) throws Exception {
|
||||
AppDetailResp appDetailResp = getAppDetailResp(request);
|
||||
authCheck(appDetailResp, queryItemReq.getIds(), ApiItemType.METRIC);
|
||||
List<SingleItemQueryResult> results = Lists.newArrayList();
|
||||
Map<Long, Item> map = appDetailResp.getConfig().getItems().stream()
|
||||
.collect(Collectors.toMap(Item::getId, i -> i));
|
||||
for (Long id : queryItemReq.getIds()) {
|
||||
Item item = map.get(id);
|
||||
SingleItemQueryResult apiQuerySingleResult = dataQuery(appDetailResp.getId(),
|
||||
item, queryItemReq.getDateConf(), queryItemReq.getLimit());
|
||||
results.add(apiQuerySingleResult);
|
||||
}
|
||||
return ItemQueryResultResp.builder().results(results).build();
|
||||
}
|
||||
|
||||
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp, DateConf dateConf, Long limit) {
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName).collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setModelIds(modelIds);
|
||||
queryStructReq.setLimit(limit);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
|
||||
MetricResp metricResp = catalog.getMetric(item.getId());
|
||||
item.setCreatedBy(metricResp.getCreatedBy());
|
||||
item.setBizName(metricResp.getBizName());
|
||||
item.setName(metricResp.getName());
|
||||
List<Item> items = item.getRelateItems();
|
||||
List<DimensionResp> dimensionResps = Lists.newArrayList();
|
||||
if (!CollectionUtils.isEmpty(items)) {
|
||||
List<Long> ids = items.stream().map(Item::getId).collect(Collectors.toList());
|
||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||
dimensionFilter.setIds(ids);
|
||||
dimensionResps = catalog.getDimensions(dimensionFilter);
|
||||
}
|
||||
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit);
|
||||
SemanticQueryResp semanticQueryResp = queryByReq(queryStructReq, User.getAppUser(appId));
|
||||
SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult();
|
||||
apiQuerySingleResult.setItem(item);
|
||||
apiQuerySingleResult.setResult(semanticQueryResp);
|
||||
return apiQuerySingleResult;
|
||||
}
|
||||
|
||||
private AppDetailResp getAppDetailResp(HttpServletRequest request) {
|
||||
int appId = Integer.parseInt(request.getHeader(ApiHeaderCheckAspect.APPID));
|
||||
return appService.getApp(appId);
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
if (Constants.MONTH.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.MONTH.getName();
|
||||
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.WEEK.getName();
|
||||
} else {
|
||||
return TimeDimensionEnum.DAY.getName();
|
||||
}
|
||||
}
|
||||
|
||||
private void authCheck(AppDetailResp appDetailResp, List<Long> ids, ApiItemType type) {
|
||||
Set<Long> idsInApp = appDetailResp.allItems().stream()
|
||||
.filter(item -> type.equals(item.getType())).map(Item::getId).collect(Collectors.toSet());
|
||||
if (!idsInApp.containsAll(ids)) {
|
||||
throw new InvalidArgumentException("查询范围超过应用申请范围, 请检查");
|
||||
}
|
||||
}
|
||||
|
||||
private ExplainResp getExplainResp(QueryStatement queryStatement) {
|
||||
String sql = "";
|
||||
if (Objects.nonNull(queryStatement)) {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.tencent.supersonic.headless.server.service.impl;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
@@ -49,6 +47,12 @@ import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
|
||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
@@ -57,11 +61,8 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -159,7 +160,7 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
|
||||
public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
|
||||
MetaFilter metaFilter = new MetaFilter();
|
||||
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
|
||||
metaFilter.setStatus(Lists.newArrayList(StatusEnum.ONLINE.getCode()));
|
||||
metaFilter.setIds(filter.getDataSetIds());
|
||||
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(metaFilter, User.getFakeUser());
|
||||
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
|
||||
@@ -209,7 +210,7 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
return modelSchemaResps;
|
||||
}
|
||||
MetaFilter metaFilter = new MetaFilter(modelIds);
|
||||
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
|
||||
metaFilter.setStatus(Lists.newArrayList(StatusEnum.ONLINE.getCode()));
|
||||
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter)
|
||||
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
|
||||
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
|
||||
|
||||
@@ -143,8 +143,12 @@
|
||||
<if test="sensitiveLevel != null">
|
||||
and t.sensitive_level = #{sensitiveLevel}
|
||||
</if>
|
||||
<if test="status != null">
|
||||
and t.status = #{status}
|
||||
<if test="status != null and status.size >0">
|
||||
and t.status in
|
||||
<foreach collection="status" index="index" item="status" open="(" close=")"
|
||||
separator=",">
|
||||
#{status}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="modelIds != null and modelIds.size >0">
|
||||
and t.model_id in
|
||||
@@ -166,6 +170,9 @@
|
||||
<if test="isTag != null and isTag == 1">
|
||||
and t1.id is not null
|
||||
</if>
|
||||
<if test="isTag != null and isTag == 0">
|
||||
and t1.id is null
|
||||
</if>
|
||||
</select>
|
||||
|
||||
|
||||
|
||||
@@ -97,6 +97,26 @@
|
||||
</foreach>
|
||||
</update>
|
||||
|
||||
<update id="batchPublish" parameterType="java.util.List">
|
||||
<foreach collection="list" item="metric" separator=";">
|
||||
update s2_metric
|
||||
set is_publish = 1,
|
||||
updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP},
|
||||
updated_by = #{metric.updatedBy,jdbcType=VARCHAR}
|
||||
where id = #{metric.id,jdbcType=BIGINT}
|
||||
</foreach>
|
||||
</update>
|
||||
|
||||
<update id="batchUnPublish" parameterType="java.util.List">
|
||||
<foreach collection="list" item="metric" separator=";">
|
||||
update s2_metric
|
||||
set is_publish = 0,
|
||||
updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP},
|
||||
updated_by = #{metric.updatedBy,jdbcType=VARCHAR}
|
||||
where id = #{metric.id,jdbcType=BIGINT}
|
||||
</foreach>
|
||||
</update>
|
||||
|
||||
<select id="query" resultMap="ResultMapWithBLOBs">
|
||||
select t.*
|
||||
from s2_metric t
|
||||
@@ -130,8 +150,12 @@
|
||||
<if test="sensitiveLevel != null">
|
||||
and t.sensitive_level = #{sensitiveLevel}
|
||||
</if>
|
||||
<if test="status != null">
|
||||
and t.status = #{status}
|
||||
<if test="status != null and status.size >0">
|
||||
and t.status in
|
||||
<foreach collection="status" index="index" item="status" open="(" close=")"
|
||||
separator=",">
|
||||
#{status}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="modelIds != null and modelIds.size >0">
|
||||
and t.model_id in
|
||||
@@ -153,6 +177,12 @@
|
||||
<if test="isTag != null and isTag == 1">
|
||||
and t1.id is not null
|
||||
</if>
|
||||
<if test="isTag != null and isTag == 0">
|
||||
and t1.id is null
|
||||
</if>
|
||||
<if test="isPublish != null and isPublish == 1">
|
||||
and (t.created_by = #{userName} or t.is_publish = 1)
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<select id="queryMetrics" resultMap="ResultMapWithBLOBs">
|
||||
|
||||
@@ -57,11 +57,6 @@ import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import com.tencent.supersonic.headless.server.service.TagObjectService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
@@ -69,6 +64,11 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class ModelDemoDataLoader {
|
||||
@@ -121,6 +121,7 @@ public class ModelDemoDataLoader {
|
||||
addDataSet_2();
|
||||
addAuthGroup_1();
|
||||
addAuthGroup_2();
|
||||
batchPushlishMetric();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to add model demo data", e);
|
||||
}
|
||||
@@ -585,6 +586,11 @@ public class ModelDemoDataLoader {
|
||||
authService.addOrUpdateAuthGroup(authGroupReq);
|
||||
}
|
||||
|
||||
private void batchPushlishMetric() {
|
||||
List<Long> ids = Lists.newArrayList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
|
||||
metricService.batchPublish(ids, User.getFakeUser());
|
||||
}
|
||||
|
||||
private RelateDimension getRelateDimension(List<Long> dimensionIds) {
|
||||
RelateDimension relateDimension = new RelateDimension();
|
||||
for (Long id : dimensionIds) {
|
||||
|
||||
@@ -288,4 +288,6 @@ CREATE TABLE IF NOT EXISTS `s2_query_rule` (
|
||||
COMMENT ON TABLE s2_query_rule IS 'tag query rule table';
|
||||
|
||||
--20240325
|
||||
alter table s2_metric change tags classifications varchar(500) null;
|
||||
alter table s2_metric change tags classifications varchar(500) null;
|
||||
alter table s2_metric add column `is_publish` int(10) DEFAULT NULL COMMENT '是否发布';
|
||||
update s2_metric set is_publish = 1;
|
||||
@@ -170,9 +170,9 @@ create table IF NOT EXISTS s2_auth_groups
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `s2_metric` (
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`model_id` INT NOT NULL ,
|
||||
`name` varchar(255) NOT NULL ,
|
||||
`id` INT NOT NULL AUTO_INCREMENT,
|
||||
`model_id` INT NOT NULL ,
|
||||
`name` varchar(255) NOT NULL ,
|
||||
`biz_name` varchar(255) NOT NULL ,
|
||||
`description` varchar(500) DEFAULT NULL ,
|
||||
`status` INT NOT NULL ,
|
||||
@@ -190,6 +190,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
|
||||
`relate_dimensions` varchar(500) DEFAULT NULL,
|
||||
`ext` LONGVARCHAR DEFAULT NULL ,
|
||||
`define_type` varchar(50) NOT NULL, -- MEASURE, FIELD, METRIC
|
||||
`is_publish` INT,
|
||||
PRIMARY KEY (`id`)
|
||||
);
|
||||
COMMENT ON TABLE s2_metric IS 'metric information table';
|
||||
|
||||
@@ -295,6 +295,7 @@ CREATE TABLE `s2_metric`
|
||||
`relate_dimensions` varchar(500) DEFAULT NULL COMMENT '指标相关维度',
|
||||
`ext` text DEFAULT NULL,
|
||||
`define_type` varchar(50) DEFAULT NULL, -- MEASURE, FIELD, METRIC
|
||||
`is_publish` int(10) DEFAULT NULL COMMENT '是否发布',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE = InnoDB
|
||||
DEFAULT CHARSET = utf8 COMMENT ='指标表';
|
||||
|
||||
Reference in New Issue
Block a user