mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 02:46:56 +00:00
(feature)(headless) Add tag query api (#790)
This commit is contained in:
@@ -19,6 +19,8 @@ public class Aggregator {
|
||||
|
||||
private List<String> args;
|
||||
|
||||
private String alias;
|
||||
|
||||
public Aggregator() {
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
@@ -35,11 +39,6 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Data
|
||||
@Slf4j
|
||||
@@ -207,7 +206,8 @@ public class QueryStructReq extends SemanticQueryReq {
|
||||
}
|
||||
sumFunction.setParameters(new ExpressionList(new Column(columnName)));
|
||||
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
|
||||
selectExpressionItem.setAlias(new Alias(columnName));
|
||||
String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
|
||||
selectExpressionItem.setAlias(new Alias(alias));
|
||||
selectItems.add(selectExpressionItem);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.tencent.supersonic.headless.api.pojo.request;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Data
|
||||
@Slf4j
|
||||
@ToString
|
||||
public class QueryTagReq extends SemanticQueryReq {
|
||||
|
||||
private List<String> groups = new ArrayList<>();
|
||||
private List<Aggregator> aggregators = new ArrayList<>();
|
||||
private List<Filter> tagFilters = new ArrayList<>();
|
||||
private List<Order> orders = new ArrayList<>();
|
||||
|
||||
private Long limit = 20L;
|
||||
private Long offset = 0L;
|
||||
|
||||
private String tagFiltersDate;
|
||||
private DateConf dateInfo;
|
||||
|
||||
@Override
|
||||
public String toCustomizedString() {
|
||||
StringBuilder stringBuilder = new StringBuilder("{");
|
||||
stringBuilder.append("\"viewId\":")
|
||||
.append(viewId);
|
||||
stringBuilder.append("\"modelIds\":")
|
||||
.append(modelIds);
|
||||
stringBuilder.append(",\"groups\":")
|
||||
.append(groups);
|
||||
stringBuilder.append(",\"aggregators\":")
|
||||
.append(aggregators);
|
||||
stringBuilder.append(",\"orders\":")
|
||||
.append(orders);
|
||||
stringBuilder.append(",\"tagFilters\":")
|
||||
.append(tagFilters);
|
||||
stringBuilder.append(",\"dateInfo\":")
|
||||
.append(dateInfo);
|
||||
stringBuilder.append(",\"params\":")
|
||||
.append(params);
|
||||
stringBuilder.append(",\"limit\":")
|
||||
.append(limit);
|
||||
stringBuilder.append('}');
|
||||
return stringBuilder.toString();
|
||||
}
|
||||
|
||||
public List<String> getMetrics() {
|
||||
List<String> metrics = Lists.newArrayList();
|
||||
if (!CollectionUtils.isEmpty(this.aggregators)) {
|
||||
metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ public class SemanticSchemaResp {
|
||||
private SchemaType schemaType;
|
||||
private List<MetricSchemaResp> metrics = Lists.newArrayList();
|
||||
private List<DimSchemaResp> dimensions = Lists.newArrayList();
|
||||
private List<TagResp> tags = Lists.newArrayList();
|
||||
private List<ModelRela> modelRelas = Lists.newArrayList();
|
||||
private List<ModelResp> modelResps = Lists.newArrayList();
|
||||
private ViewResp viewResp;
|
||||
|
||||
@@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
@@ -29,14 +37,6 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Aspect
|
||||
@Component
|
||||
@Slf4j
|
||||
@@ -63,9 +63,17 @@ public class DimValueAspect {
|
||||
if (queryReq instanceof QuerySqlReq) {
|
||||
return handleSqlDimValue(joinPoint);
|
||||
}
|
||||
|
||||
if (queryReq instanceof QueryTagReq) {
|
||||
return handleTagValue(joinPoint);
|
||||
}
|
||||
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
|
||||
}
|
||||
|
||||
public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
return (SemanticQueryResp) joinPoint.proceed();
|
||||
}
|
||||
|
||||
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
Object[] args = joinPoint.getArgs();
|
||||
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
||||
|
||||
@@ -52,6 +52,7 @@ public class ModelYamlManager {
|
||||
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
||||
}
|
||||
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
|
||||
dataModelYamlTpl.setId(modelResp.getId());
|
||||
return dataModelYamlTpl;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.Field;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
||||
@@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
@@ -44,11 +40,16 @@ import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SemanticSchemaManager {
|
||||
|
||||
private final Catalog catalog;
|
||||
|
||||
public SemanticSchemaManager(Catalog catalog) {
|
||||
@@ -87,6 +88,54 @@ public class SemanticSchemaManager {
|
||||
return semanticModel;
|
||||
}
|
||||
|
||||
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
|
||||
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
|
||||
throw new Exception("semanticSchemaResp tag is empty");
|
||||
}
|
||||
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
|
||||
//Map<String, List<Dimension>> dimensions = new HashMap<>();
|
||||
Map<Long, List<TagResp>> tagMap = new HashMap<>();
|
||||
for (TagResp tagResp : semanticSchemaResp.getTags()) {
|
||||
if (!tagMap.containsKey(tagResp.getModelId())) {
|
||||
tagMap.put(tagResp.getModelId(), new ArrayList<>());
|
||||
}
|
||||
tagMap.get(tagResp.getModelId()).add(tagResp);
|
||||
}
|
||||
if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
|
||||
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
List<String> tagNames = new ArrayList<>();
|
||||
if (tagMap.containsKey(entry.getValue().getId())) {
|
||||
for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
|
||||
tagNames.add(tagResp.getBizName());
|
||||
Dimension dimension = Dimension.builder().build();
|
||||
dimension.setType("");
|
||||
dimension.setExpr(tagResp.getExpr());
|
||||
dimension.setName(tagResp.getBizName());
|
||||
dimension.setOwners("");
|
||||
dimension.setBizName(tagResp.getBizName());
|
||||
if (Objects.isNull(dimension.getDataType())) {
|
||||
dimension.setDataType(DataType.UNKNOWN);
|
||||
}
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
|
||||
dimensions.add(dimension);
|
||||
}
|
||||
}
|
||||
if (semanticModel.getDimensionMap().containsKey(entry.getKey())) {
|
||||
semanticModel.getDimensionMap().get(entry.getKey()).stream()
|
||||
.filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> {
|
||||
dimensions.add(d);
|
||||
});
|
||||
}
|
||||
semanticModel.getDimensionMap().put(entry.getKey(), dimensions);
|
||||
}
|
||||
}
|
||||
// metric ignored
|
||||
semanticModel.setMetrics(new ArrayList<>());
|
||||
return semanticModel;
|
||||
}
|
||||
|
||||
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
||||
return getMetricsByMetricYamlTpl(t);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
@@ -58,6 +59,14 @@ public class QueryController {
|
||||
return queryService.queryByReq(querySqlReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/tag")
|
||||
public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryByReq(queryTagReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/queryMetricDataById")
|
||||
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
|
||||
HttpServletRequest request) throws Exception {
|
||||
|
||||
@@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
|
||||
@@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
@@ -83,6 +85,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
private StatUtils statUtils;
|
||||
private final QueryUtils queryUtils;
|
||||
private final QueryReqConverter queryReqConverter;
|
||||
private final TagReqConverter tagReqConverter;
|
||||
private final Catalog catalog;
|
||||
private final AppService appService;
|
||||
private final QueryCache queryCache;
|
||||
@@ -102,7 +105,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
StatUtils statUtils,
|
||||
QueryUtils queryUtils,
|
||||
QueryReqConverter queryReqConverter,
|
||||
Catalog catalog,
|
||||
TagReqConverter tagReqConverter, Catalog catalog,
|
||||
AppService appService,
|
||||
QueryCache queryCache,
|
||||
SemanticSchemaManager semanticSchemaManager,
|
||||
@@ -114,6 +117,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
this.statUtils = statUtils;
|
||||
this.queryUtils = queryUtils;
|
||||
this.queryReqConverter = queryReqConverter;
|
||||
this.tagReqConverter = tagReqConverter;
|
||||
this.catalog = catalog;
|
||||
this.appService = appService;
|
||||
this.queryCache = queryCache;
|
||||
@@ -185,6 +189,9 @@ public class QueryServiceImpl implements QueryService {
|
||||
if (semanticQueryReq instanceof QueryMultiStructReq) {
|
||||
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
|
||||
}
|
||||
if (semanticQueryReq instanceof QueryTagReq) {
|
||||
return buildTagQueryStatement((QueryTagReq) semanticQueryReq);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -220,6 +227,21 @@ public class QueryServiceImpl implements QueryService {
|
||||
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
|
||||
}
|
||||
|
||||
private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq)
|
||||
throws Exception {
|
||||
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
|
||||
SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq);
|
||||
schemaFilterReq.setModelIds(queryTagReq.getModelIds());
|
||||
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
|
||||
QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp);
|
||||
queryStatement.setModelIds(queryTagReq.getModelIds());
|
||||
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
|
||||
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
|
||||
SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp);
|
||||
queryStatement.setSemanticModel(semanticModel);
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
|
||||
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
|
||||
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
|
||||
|
||||
@@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.TagFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.DomainService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||
import com.tencent.supersonic.headless.server.service.TagService;
|
||||
import com.tencent.supersonic.headless.server.service.ViewService;
|
||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
@@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
private final DomainService domainService;
|
||||
private final ViewService viewService;
|
||||
private final ModelRelaService modelRelaService;
|
||||
private final TagService tagService;
|
||||
|
||||
public SchemaServiceImpl(ModelService modelService,
|
||||
DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
DomainService domainService,
|
||||
ViewService viewService,
|
||||
ModelRelaService modelRelaService,
|
||||
StatUtils statUtils) {
|
||||
DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
DomainService domainService,
|
||||
ViewService viewService,
|
||||
ModelRelaService modelRelaService,
|
||||
StatUtils statUtils, TagService tagService) {
|
||||
this.modelService = modelService;
|
||||
this.dimensionService = dimensionService;
|
||||
this.metricService = metricService;
|
||||
@@ -93,6 +97,7 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
this.viewService = viewService;
|
||||
this.modelRelaService = modelRelaService;
|
||||
this.statUtils = statUtils;
|
||||
this.tagService = tagService;
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@@ -301,6 +306,11 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
.flatMap(Collection::stream).collect(Collectors.toList()));
|
||||
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
|
||||
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
|
||||
// add tag info
|
||||
TagFilter tagFilter = new TagFilter();
|
||||
tagFilter.setModelIds(schemaFilterReq.getModelIds());
|
||||
List<TagResp> tagResps = tagService.query(tagFilter);
|
||||
semanticSchemaResp.setTags(tagResps);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
|
||||
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));
|
||||
|
||||
@@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
|
||||
@Component
|
||||
@@ -96,6 +96,48 @@ public class StatUtils {
|
||||
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
|
||||
initStructStatInfo(queryStructCmd, facadeUser);
|
||||
}
|
||||
if (semanticQueryReq instanceof QueryTagReq) {
|
||||
initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser);
|
||||
}
|
||||
}
|
||||
|
||||
public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) {
|
||||
QueryStat queryStatInfo = new QueryStat();
|
||||
String traceId = "";
|
||||
List<String> dimensions = queryTagReq.getGroups();
|
||||
|
||||
List<String> metrics = new ArrayList<>();
|
||||
queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
|
||||
String user = getUserName(facadeUser);
|
||||
|
||||
try {
|
||||
queryStatInfo.setTraceId(traceId)
|
||||
.setViewId(queryTagReq.getViewId())
|
||||
.setUser(user)
|
||||
.setQueryType(QueryType.STRUCT.getValue())
|
||||
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
|
||||
.setQueryStructCmd(queryTagReq.toString())
|
||||
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString()))
|
||||
.setStartTime(System.currentTimeMillis())
|
||||
.setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators()))
|
||||
.setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups()))
|
||||
.setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators()))
|
||||
.setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders()))
|
||||
.setFilterCols(objectMapper.writeValueAsString(
|
||||
sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters())))
|
||||
.setUseResultCache(true)
|
||||
.setUseSqlCache(true)
|
||||
.setMetrics(objectMapper.writeValueAsString(metrics))
|
||||
.setDimensions(objectMapper.writeValueAsString(dimensions))
|
||||
.setQueryOptMode(QueryOptMode.NONE.name());
|
||||
if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) {
|
||||
queryStatInfo.setModelId(queryTagReq.getModelIds().get(0));
|
||||
}
|
||||
} catch (JsonProcessingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
StatUtils.set(queryStatInfo);
|
||||
|
||||
}
|
||||
|
||||
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.tencent.supersonic.headless.server.utils;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricTable;
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
|
||||
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class TagReqConverter {
|
||||
|
||||
@Value("${query.sql.limitWrapper:true}")
|
||||
private Boolean limitWrapper;
|
||||
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
|
||||
@Autowired
|
||||
private SqlGenerateUtils sqlGenerateUtils;
|
||||
|
||||
public QueryStatement convert(QueryTagReq queryTagReq,
|
||||
SemanticSchemaResp semanticSchemaResp) throws Exception {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
// covert to QueryReqConverter
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
BeanUtils.copyProperties(queryTagReq, queryStructReq);
|
||||
if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) {
|
||||
queryStructReq.setDimensionFilters(queryTagReq.getTagFilters());
|
||||
}
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert();
|
||||
if (Objects.nonNull(querySqlReq)) {
|
||||
log.info("convert to QuerySqlReq {}", querySqlReq);
|
||||
String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql());
|
||||
MetricTable metricTable = new MetricTable();
|
||||
metricTable.setMetrics(new ArrayList<>());
|
||||
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
|
||||
semanticSchemaResp.getModelResps().get(0).getBizName()));
|
||||
metricTable.setAggOption(AggOption.NATIVE);
|
||||
List<String> allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql());
|
||||
metricTable.setDimensions(allFields);
|
||||
metricTable.setAlias(tableName.toLowerCase());
|
||||
List<MetricTable> tables = new ArrayList<>();
|
||||
tables.add(metricTable);
|
||||
//.build ParseSqlReq
|
||||
ViewQueryParam result = new ViewQueryParam();
|
||||
BeanUtils.copyProperties(querySqlReq, result);
|
||||
result.setTables(tables);
|
||||
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
|
||||
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
|
||||
database.getVersion())) {
|
||||
result.setSupportWith(false);
|
||||
result.setWithAlias(false);
|
||||
}
|
||||
//.physicalSql by ParseSqlReq
|
||||
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
|
||||
queryStructReq.setViewId(querySqlReq.getViewId());
|
||||
queryStructReq.setQueryType(QueryType.TAG);
|
||||
QueryParam queryParam = new QueryParam();
|
||||
convert(queryTagReq, queryParam);
|
||||
queryStatement.setQueryParam(queryParam);
|
||||
queryStatement.setViewQueryParam(result);
|
||||
queryStatement.setIsS2SQL(true);
|
||||
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
|
||||
queryStatement.setViewId(queryTagReq.getViewId());
|
||||
queryStatement.setEnableLimitWrapper(limitWrapper);
|
||||
}
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
public void convert(QueryTagReq queryTagReq, QueryParam queryParam) {
|
||||
BeanUtils.copyProperties(queryTagReq, queryParam);
|
||||
queryParam.setOrders(queryTagReq.getOrders());
|
||||
queryParam.setMetrics(queryTagReq.getMetrics());
|
||||
queryParam.setGroups(queryTagReq.getGroups());
|
||||
queryParam.setDimensionFilters(queryTagReq.getTagFilters());
|
||||
queryParam.setQueryType(QueryType.TAG);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user