mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 11:07:06 +00:00
(feature)(headless) Add tag query api (#790)
This commit is contained in:
@@ -19,6 +19,8 @@ public class Aggregator {
|
|||||||
|
|
||||||
private List<String> args;
|
private List<String> args;
|
||||||
|
|
||||||
|
private String alias;
|
||||||
|
|
||||||
public Aggregator() {
|
public Aggregator() {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
|
|||||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import net.sf.jsqlparser.JSQLParserException;
|
import net.sf.jsqlparser.JSQLParserException;
|
||||||
@@ -35,11 +39,6 @@ import org.apache.commons.lang3.StringUtils;
|
|||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@@ -207,7 +206,8 @@ public class QueryStructReq extends SemanticQueryReq {
|
|||||||
}
|
}
|
||||||
sumFunction.setParameters(new ExpressionList(new Column(columnName)));
|
sumFunction.setParameters(new ExpressionList(new Column(columnName)));
|
||||||
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
|
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
|
||||||
selectExpressionItem.setAlias(new Alias(columnName));
|
String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
|
||||||
|
selectExpressionItem.setAlias(new Alias(alias));
|
||||||
selectItems.add(selectExpressionItem);
|
selectItems.add(selectExpressionItem);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package com.tencent.supersonic.headless.api.pojo.request;
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.Filter;
|
||||||
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.ToString;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Slf4j
|
||||||
|
@ToString
|
||||||
|
public class QueryTagReq extends SemanticQueryReq {
|
||||||
|
|
||||||
|
private List<String> groups = new ArrayList<>();
|
||||||
|
private List<Aggregator> aggregators = new ArrayList<>();
|
||||||
|
private List<Filter> tagFilters = new ArrayList<>();
|
||||||
|
private List<Order> orders = new ArrayList<>();
|
||||||
|
|
||||||
|
private Long limit = 20L;
|
||||||
|
private Long offset = 0L;
|
||||||
|
|
||||||
|
private String tagFiltersDate;
|
||||||
|
private DateConf dateInfo;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toCustomizedString() {
|
||||||
|
StringBuilder stringBuilder = new StringBuilder("{");
|
||||||
|
stringBuilder.append("\"viewId\":")
|
||||||
|
.append(viewId);
|
||||||
|
stringBuilder.append("\"modelIds\":")
|
||||||
|
.append(modelIds);
|
||||||
|
stringBuilder.append(",\"groups\":")
|
||||||
|
.append(groups);
|
||||||
|
stringBuilder.append(",\"aggregators\":")
|
||||||
|
.append(aggregators);
|
||||||
|
stringBuilder.append(",\"orders\":")
|
||||||
|
.append(orders);
|
||||||
|
stringBuilder.append(",\"tagFilters\":")
|
||||||
|
.append(tagFilters);
|
||||||
|
stringBuilder.append(",\"dateInfo\":")
|
||||||
|
.append(dateInfo);
|
||||||
|
stringBuilder.append(",\"params\":")
|
||||||
|
.append(params);
|
||||||
|
stringBuilder.append(",\"limit\":")
|
||||||
|
.append(limit);
|
||||||
|
stringBuilder.append('}');
|
||||||
|
return stringBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getMetrics() {
|
||||||
|
List<String> metrics = Lists.newArrayList();
|
||||||
|
if (!CollectionUtils.isEmpty(this.aggregators)) {
|
||||||
|
metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,7 @@ public class SemanticSchemaResp {
|
|||||||
private SchemaType schemaType;
|
private SchemaType schemaType;
|
||||||
private List<MetricSchemaResp> metrics = Lists.newArrayList();
|
private List<MetricSchemaResp> metrics = Lists.newArrayList();
|
||||||
private List<DimSchemaResp> dimensions = Lists.newArrayList();
|
private List<DimSchemaResp> dimensions = Lists.newArrayList();
|
||||||
|
private List<TagResp> tags = Lists.newArrayList();
|
||||||
private List<ModelRela> modelRelas = Lists.newArrayList();
|
private List<ModelRela> modelRelas = Lists.newArrayList();
|
||||||
private List<ModelResp> modelResps = Lists.newArrayList();
|
private List<ModelResp> modelResps = Lists.newArrayList();
|
||||||
private ViewResp viewResp;
|
private ViewResp viewResp;
|
||||||
|
|||||||
@@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
@@ -29,14 +37,6 @@ import org.springframework.beans.factory.annotation.Value;
|
|||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Aspect
|
@Aspect
|
||||||
@Component
|
@Component
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@@ -63,9 +63,17 @@ public class DimValueAspect {
|
|||||||
if (queryReq instanceof QuerySqlReq) {
|
if (queryReq instanceof QuerySqlReq) {
|
||||||
return handleSqlDimValue(joinPoint);
|
return handleSqlDimValue(joinPoint);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (queryReq instanceof QueryTagReq) {
|
||||||
|
return handleTagValue(joinPoint);
|
||||||
|
}
|
||||||
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
|
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||||
|
return (SemanticQueryResp) joinPoint.proceed();
|
||||||
|
}
|
||||||
|
|
||||||
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||||
Object[] args = joinPoint.getArgs();
|
Object[] args = joinPoint.getArgs();
|
||||||
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ public class ModelYamlManager {
|
|||||||
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
||||||
}
|
}
|
||||||
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
|
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
|
||||||
|
dataModelYamlTpl.setId(modelResp.getId());
|
||||||
return dataModelYamlTpl;
|
return dataModelYamlTpl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.Field;
|
import com.tencent.supersonic.headless.api.pojo.Field;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
||||||
@@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
|
|||||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
||||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||||
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
|
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.tuple.Triple;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
@@ -44,11 +40,16 @@ import java.util.Objects;
|
|||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.tuple.Triple;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
public class SemanticSchemaManager {
|
public class SemanticSchemaManager {
|
||||||
|
|
||||||
private final Catalog catalog;
|
private final Catalog catalog;
|
||||||
|
|
||||||
public SemanticSchemaManager(Catalog catalog) {
|
public SemanticSchemaManager(Catalog catalog) {
|
||||||
@@ -87,6 +88,54 @@ public class SemanticSchemaManager {
|
|||||||
return semanticModel;
|
return semanticModel;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
|
||||||
|
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
|
||||||
|
throw new Exception("semanticSchemaResp tag is empty");
|
||||||
|
}
|
||||||
|
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
|
||||||
|
//Map<String, List<Dimension>> dimensions = new HashMap<>();
|
||||||
|
Map<Long, List<TagResp>> tagMap = new HashMap<>();
|
||||||
|
for (TagResp tagResp : semanticSchemaResp.getTags()) {
|
||||||
|
if (!tagMap.containsKey(tagResp.getModelId())) {
|
||||||
|
tagMap.put(tagResp.getModelId(), new ArrayList<>());
|
||||||
|
}
|
||||||
|
tagMap.get(tagResp.getModelId()).add(tagResp);
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
|
||||||
|
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
|
||||||
|
List<Dimension> dimensions = new ArrayList<>();
|
||||||
|
List<String> tagNames = new ArrayList<>();
|
||||||
|
if (tagMap.containsKey(entry.getValue().getId())) {
|
||||||
|
for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
|
||||||
|
tagNames.add(tagResp.getBizName());
|
||||||
|
Dimension dimension = Dimension.builder().build();
|
||||||
|
dimension.setType("");
|
||||||
|
dimension.setExpr(tagResp.getExpr());
|
||||||
|
dimension.setName(tagResp.getBizName());
|
||||||
|
dimension.setOwners("");
|
||||||
|
dimension.setBizName(tagResp.getBizName());
|
||||||
|
if (Objects.isNull(dimension.getDataType())) {
|
||||||
|
dimension.setDataType(DataType.UNKNOWN);
|
||||||
|
}
|
||||||
|
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||||
|
dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
|
||||||
|
dimensions.add(dimension);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (semanticModel.getDimensionMap().containsKey(entry.getKey())) {
|
||||||
|
semanticModel.getDimensionMap().get(entry.getKey()).stream()
|
||||||
|
.filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> {
|
||||||
|
dimensions.add(d);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
semanticModel.getDimensionMap().put(entry.getKey(), dimensions);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// metric ignored
|
||||||
|
semanticModel.setMetrics(new ArrayList<>());
|
||||||
|
return semanticModel;
|
||||||
|
}
|
||||||
|
|
||||||
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
||||||
return getMetricsByMetricYamlTpl(t);
|
return getMetricsByMetricYamlTpl(t);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||||
@@ -58,6 +59,14 @@ public class QueryController {
|
|||||||
return queryService.queryByReq(querySqlReq, user);
|
return queryService.queryByReq(querySqlReq, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tag")
|
||||||
|
public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
|
||||||
|
HttpServletRequest request,
|
||||||
|
HttpServletResponse response) throws Exception {
|
||||||
|
User user = UserHolder.findUser(request, response);
|
||||||
|
return queryService.queryByReq(queryTagReq, user);
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/queryMetricDataById")
|
@PostMapping("/queryMetricDataById")
|
||||||
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
|
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
|
||||||
HttpServletRequest request) throws Exception {
|
HttpServletRequest request) throws Exception {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
|
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
|
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
|
||||||
@@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
|
|||||||
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
|
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
|
||||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||||
|
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
@@ -83,6 +85,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
private StatUtils statUtils;
|
private StatUtils statUtils;
|
||||||
private final QueryUtils queryUtils;
|
private final QueryUtils queryUtils;
|
||||||
private final QueryReqConverter queryReqConverter;
|
private final QueryReqConverter queryReqConverter;
|
||||||
|
private final TagReqConverter tagReqConverter;
|
||||||
private final Catalog catalog;
|
private final Catalog catalog;
|
||||||
private final AppService appService;
|
private final AppService appService;
|
||||||
private final QueryCache queryCache;
|
private final QueryCache queryCache;
|
||||||
@@ -102,7 +105,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
StatUtils statUtils,
|
StatUtils statUtils,
|
||||||
QueryUtils queryUtils,
|
QueryUtils queryUtils,
|
||||||
QueryReqConverter queryReqConverter,
|
QueryReqConverter queryReqConverter,
|
||||||
Catalog catalog,
|
TagReqConverter tagReqConverter, Catalog catalog,
|
||||||
AppService appService,
|
AppService appService,
|
||||||
QueryCache queryCache,
|
QueryCache queryCache,
|
||||||
SemanticSchemaManager semanticSchemaManager,
|
SemanticSchemaManager semanticSchemaManager,
|
||||||
@@ -114,6 +117,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
this.statUtils = statUtils;
|
this.statUtils = statUtils;
|
||||||
this.queryUtils = queryUtils;
|
this.queryUtils = queryUtils;
|
||||||
this.queryReqConverter = queryReqConverter;
|
this.queryReqConverter = queryReqConverter;
|
||||||
|
this.tagReqConverter = tagReqConverter;
|
||||||
this.catalog = catalog;
|
this.catalog = catalog;
|
||||||
this.appService = appService;
|
this.appService = appService;
|
||||||
this.queryCache = queryCache;
|
this.queryCache = queryCache;
|
||||||
@@ -185,6 +189,9 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
if (semanticQueryReq instanceof QueryMultiStructReq) {
|
if (semanticQueryReq instanceof QueryMultiStructReq) {
|
||||||
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
|
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
|
||||||
}
|
}
|
||||||
|
if (semanticQueryReq instanceof QueryTagReq) {
|
||||||
|
return buildTagQueryStatement((QueryTagReq) semanticQueryReq);
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,6 +227,21 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
|
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq)
|
||||||
|
throws Exception {
|
||||||
|
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
|
||||||
|
SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq);
|
||||||
|
schemaFilterReq.setModelIds(queryTagReq.getModelIds());
|
||||||
|
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
|
||||||
|
QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp);
|
||||||
|
queryStatement.setModelIds(queryTagReq.getModelIds());
|
||||||
|
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
|
||||||
|
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
|
||||||
|
SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp);
|
||||||
|
queryStatement.setSemanticModel(semanticModel);
|
||||||
|
return queryStatement;
|
||||||
|
}
|
||||||
|
|
||||||
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
|
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
|
||||||
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
|
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
|
||||||
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
|
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
|
||||||
|
|||||||
@@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
|
||||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||||
|
import com.tencent.supersonic.headless.server.pojo.TagFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
import com.tencent.supersonic.headless.server.service.DomainService;
|
import com.tencent.supersonic.headless.server.service.DomainService;
|
||||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
||||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.TagService;
|
||||||
import com.tencent.supersonic.headless.server.service.ViewService;
|
import com.tencent.supersonic.headless.server.service.ViewService;
|
||||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||||
@@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService {
|
|||||||
private final DomainService domainService;
|
private final DomainService domainService;
|
||||||
private final ViewService viewService;
|
private final ViewService viewService;
|
||||||
private final ModelRelaService modelRelaService;
|
private final ModelRelaService modelRelaService;
|
||||||
|
private final TagService tagService;
|
||||||
|
|
||||||
public SchemaServiceImpl(ModelService modelService,
|
public SchemaServiceImpl(ModelService modelService,
|
||||||
DimensionService dimensionService,
|
DimensionService dimensionService,
|
||||||
MetricService metricService,
|
MetricService metricService,
|
||||||
DomainService domainService,
|
DomainService domainService,
|
||||||
ViewService viewService,
|
ViewService viewService,
|
||||||
ModelRelaService modelRelaService,
|
ModelRelaService modelRelaService,
|
||||||
StatUtils statUtils) {
|
StatUtils statUtils, TagService tagService) {
|
||||||
this.modelService = modelService;
|
this.modelService = modelService;
|
||||||
this.dimensionService = dimensionService;
|
this.dimensionService = dimensionService;
|
||||||
this.metricService = metricService;
|
this.metricService = metricService;
|
||||||
@@ -93,6 +97,7 @@ public class SchemaServiceImpl implements SchemaService {
|
|||||||
this.viewService = viewService;
|
this.viewService = viewService;
|
||||||
this.modelRelaService = modelRelaService;
|
this.modelRelaService = modelRelaService;
|
||||||
this.statUtils = statUtils;
|
this.statUtils = statUtils;
|
||||||
|
this.tagService = tagService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
@@ -301,6 +306,11 @@ public class SchemaServiceImpl implements SchemaService {
|
|||||||
.flatMap(Collection::stream).collect(Collectors.toList()));
|
.flatMap(Collection::stream).collect(Collectors.toList()));
|
||||||
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
|
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
|
||||||
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
|
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
|
||||||
|
// add tag info
|
||||||
|
TagFilter tagFilter = new TagFilter();
|
||||||
|
tagFilter.setModelIds(schemaFilterReq.getModelIds());
|
||||||
|
List<TagResp> tagResps = tagService.query(tagFilter);
|
||||||
|
semanticSchemaResp.setTags(tagResps);
|
||||||
}
|
}
|
||||||
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
|
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
|
||||||
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));
|
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));
|
||||||
|
|||||||
@@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||||
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
|
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
|
||||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.codec.digest.DigestUtils;
|
|
||||||
import org.apache.logging.log4j.util.Strings;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.CompletableFuture;
|
import java.util.concurrent.CompletableFuture;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.codec.digest.DigestUtils;
|
||||||
|
import org.apache.logging.log4j.util.Strings;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@@ -96,6 +96,48 @@ public class StatUtils {
|
|||||||
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
|
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
|
||||||
initStructStatInfo(queryStructCmd, facadeUser);
|
initStructStatInfo(queryStructCmd, facadeUser);
|
||||||
}
|
}
|
||||||
|
if (semanticQueryReq instanceof QueryTagReq) {
|
||||||
|
initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) {
|
||||||
|
QueryStat queryStatInfo = new QueryStat();
|
||||||
|
String traceId = "";
|
||||||
|
List<String> dimensions = queryTagReq.getGroups();
|
||||||
|
|
||||||
|
List<String> metrics = new ArrayList<>();
|
||||||
|
queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
|
||||||
|
String user = getUserName(facadeUser);
|
||||||
|
|
||||||
|
try {
|
||||||
|
queryStatInfo.setTraceId(traceId)
|
||||||
|
.setViewId(queryTagReq.getViewId())
|
||||||
|
.setUser(user)
|
||||||
|
.setQueryType(QueryType.STRUCT.getValue())
|
||||||
|
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
|
||||||
|
.setQueryStructCmd(queryTagReq.toString())
|
||||||
|
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString()))
|
||||||
|
.setStartTime(System.currentTimeMillis())
|
||||||
|
.setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators()))
|
||||||
|
.setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups()))
|
||||||
|
.setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators()))
|
||||||
|
.setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders()))
|
||||||
|
.setFilterCols(objectMapper.writeValueAsString(
|
||||||
|
sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters())))
|
||||||
|
.setUseResultCache(true)
|
||||||
|
.setUseSqlCache(true)
|
||||||
|
.setMetrics(objectMapper.writeValueAsString(metrics))
|
||||||
|
.setDimensions(objectMapper.writeValueAsString(dimensions))
|
||||||
|
.setQueryOptMode(QueryOptMode.NONE.name());
|
||||||
|
if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) {
|
||||||
|
queryStatInfo.setModelId(queryTagReq.getModelIds().get(0));
|
||||||
|
}
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
StatUtils.set(queryStatInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
|
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
package com.tencent.supersonic.headless.server.utils;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
|
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricTable;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
|
||||||
|
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.springframework.beans.BeanUtils;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@Slf4j
|
||||||
|
public class TagReqConverter {
|
||||||
|
|
||||||
|
@Value("${query.sql.limitWrapper:true}")
|
||||||
|
private Boolean limitWrapper;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private QueryStructUtils queryStructUtils;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private SqlGenerateUtils sqlGenerateUtils;
|
||||||
|
|
||||||
|
public QueryStatement convert(QueryTagReq queryTagReq,
|
||||||
|
SemanticSchemaResp semanticSchemaResp) throws Exception {
|
||||||
|
QueryStatement queryStatement = new QueryStatement();
|
||||||
|
// covert to QueryReqConverter
|
||||||
|
QueryStructReq queryStructReq = new QueryStructReq();
|
||||||
|
BeanUtils.copyProperties(queryTagReq, queryStructReq);
|
||||||
|
if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) {
|
||||||
|
queryStructReq.setDimensionFilters(queryTagReq.getTagFilters());
|
||||||
|
}
|
||||||
|
QuerySqlReq querySqlReq = queryStructReq.convert();
|
||||||
|
if (Objects.nonNull(querySqlReq)) {
|
||||||
|
log.info("convert to QuerySqlReq {}", querySqlReq);
|
||||||
|
String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql());
|
||||||
|
MetricTable metricTable = new MetricTable();
|
||||||
|
metricTable.setMetrics(new ArrayList<>());
|
||||||
|
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
|
||||||
|
semanticSchemaResp.getModelResps().get(0).getBizName()));
|
||||||
|
metricTable.setAggOption(AggOption.NATIVE);
|
||||||
|
List<String> allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql());
|
||||||
|
metricTable.setDimensions(allFields);
|
||||||
|
metricTable.setAlias(tableName.toLowerCase());
|
||||||
|
List<MetricTable> tables = new ArrayList<>();
|
||||||
|
tables.add(metricTable);
|
||||||
|
//.build ParseSqlReq
|
||||||
|
ViewQueryParam result = new ViewQueryParam();
|
||||||
|
BeanUtils.copyProperties(querySqlReq, result);
|
||||||
|
result.setTables(tables);
|
||||||
|
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
|
||||||
|
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
|
||||||
|
database.getVersion())) {
|
||||||
|
result.setSupportWith(false);
|
||||||
|
result.setWithAlias(false);
|
||||||
|
}
|
||||||
|
//.physicalSql by ParseSqlReq
|
||||||
|
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
|
||||||
|
queryStructReq.setViewId(querySqlReq.getViewId());
|
||||||
|
queryStructReq.setQueryType(QueryType.TAG);
|
||||||
|
QueryParam queryParam = new QueryParam();
|
||||||
|
convert(queryTagReq, queryParam);
|
||||||
|
queryStatement.setQueryParam(queryParam);
|
||||||
|
queryStatement.setViewQueryParam(result);
|
||||||
|
queryStatement.setIsS2SQL(true);
|
||||||
|
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
|
||||||
|
queryStatement.setViewId(queryTagReq.getViewId());
|
||||||
|
queryStatement.setEnableLimitWrapper(limitWrapper);
|
||||||
|
}
|
||||||
|
return queryStatement;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void convert(QueryTagReq queryTagReq, QueryParam queryParam) {
|
||||||
|
BeanUtils.copyProperties(queryTagReq, queryParam);
|
||||||
|
queryParam.setOrders(queryTagReq.getOrders());
|
||||||
|
queryParam.setMetrics(queryTagReq.getMetrics());
|
||||||
|
queryParam.setGroups(queryTagReq.getGroups());
|
||||||
|
queryParam.setDimensionFilters(queryTagReq.getTagFilters());
|
||||||
|
queryParam.setQueryType(QueryType.TAG);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user