(feature)(headless) Add tag query api (#790)

This commit is contained in:
jipeli
2024-03-06 11:41:47 +08:00
committed by GitHub
parent 78d724ea83
commit 40ea6a9396
12 changed files with 337 additions and 32 deletions

View File

@@ -19,6 +19,8 @@ public class Aggregator {
private List<String> args; private List<String> args;
private String alias;
public Aggregator() { public Aggregator() {
} }

View File

@@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.JSQLParserException;
@@ -35,11 +39,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Data @Data
@Slf4j @Slf4j
@@ -207,7 +206,8 @@ public class QueryStructReq extends SemanticQueryReq {
} }
sumFunction.setParameters(new ExpressionList(new Column(columnName))); sumFunction.setParameters(new ExpressionList(new Column(columnName)));
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction); SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
selectExpressionItem.setAlias(new Alias(columnName)); String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
selectExpressionItem.setAlias(new Alias(alias));
selectItems.add(selectExpressionItem); selectItems.add(selectExpressionItem);
} }
} }

View File

@@ -0,0 +1,64 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Data
@Slf4j
@ToString
public class QueryTagReq extends SemanticQueryReq {
private List<String> groups = new ArrayList<>();
private List<Aggregator> aggregators = new ArrayList<>();
private List<Filter> tagFilters = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
private Long limit = 20L;
private Long offset = 0L;
private String tagFiltersDate;
private DateConf dateInfo;
@Override
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"viewId\":")
.append(viewId);
stringBuilder.append("\"modelIds\":")
.append(modelIds);
stringBuilder.append(",\"groups\":")
.append(groups);
stringBuilder.append(",\"aggregators\":")
.append(aggregators);
stringBuilder.append(",\"orders\":")
.append(orders);
stringBuilder.append(",\"tagFilters\":")
.append(tagFilters);
stringBuilder.append(",\"dateInfo\":")
.append(dateInfo);
stringBuilder.append(",\"params\":")
.append(params);
stringBuilder.append(",\"limit\":")
.append(limit);
stringBuilder.append('}');
return stringBuilder.toString();
}
public List<String> getMetrics() {
List<String> metrics = Lists.newArrayList();
if (!CollectionUtils.isEmpty(this.aggregators)) {
metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
}
return metrics;
}
}

View File

@@ -22,6 +22,7 @@ public class SemanticSchemaResp {
private SchemaType schemaType; private SchemaType schemaType;
private List<MetricSchemaResp> metrics = Lists.newArrayList(); private List<MetricSchemaResp> metrics = Lists.newArrayList();
private List<DimSchemaResp> dimensions = Lists.newArrayList(); private List<DimSchemaResp> dimensions = Lists.newArrayList();
private List<TagResp> tags = Lists.newArrayList();
private List<ModelRela> modelRelas = Lists.newArrayList(); private List<ModelRela> modelRelas = Lists.newArrayList();
private List<ModelResp> modelResps = Lists.newArrayList(); private List<ModelResp> modelResps = Lists.newArrayList();
private ViewResp viewResp; private ViewResp viewResp;

View File

@@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
@@ -29,14 +37,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Aspect @Aspect
@Component @Component
@Slf4j @Slf4j
@@ -63,9 +63,17 @@ public class DimValueAspect {
if (queryReq instanceof QuerySqlReq) { if (queryReq instanceof QuerySqlReq) {
return handleSqlDimValue(joinPoint); return handleSqlDimValue(joinPoint);
} }
if (queryReq instanceof QueryTagReq) {
return handleTagValue(joinPoint);
}
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
} }
public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable {
return (SemanticQueryResp) joinPoint.proceed();
}
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable { private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs(); Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0]; QueryStructReq queryStructReq = (QueryStructReq) args[0];

View File

@@ -52,6 +52,7 @@ public class ModelYamlManager {
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
} }
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
dataModelYamlTpl.setId(modelResp.getId());
return dataModelYamlTpl; return dataModelYamlTpl;
} }

View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
@@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@@ -44,11 +40,16 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j @Slf4j
@Service @Service
public class SemanticSchemaManager { public class SemanticSchemaManager {
private final Catalog catalog; private final Catalog catalog;
public SemanticSchemaManager(Catalog catalog) { public SemanticSchemaManager(Catalog catalog) {
@@ -87,6 +88,54 @@ public class SemanticSchemaManager {
return semanticModel; return semanticModel;
} }
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
throw new Exception("semanticSchemaResp tag is empty");
}
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
//Map<String, List<Dimension>> dimensions = new HashMap<>();
Map<Long, List<TagResp>> tagMap = new HashMap<>();
for (TagResp tagResp : semanticSchemaResp.getTags()) {
if (!tagMap.containsKey(tagResp.getModelId())) {
tagMap.put(tagResp.getModelId(), new ArrayList<>());
}
tagMap.get(tagResp.getModelId()).add(tagResp);
}
if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
List<Dimension> dimensions = new ArrayList<>();
List<String> tagNames = new ArrayList<>();
if (tagMap.containsKey(entry.getValue().getId())) {
for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
tagNames.add(tagResp.getBizName());
Dimension dimension = Dimension.builder().build();
dimension.setType("");
dimension.setExpr(tagResp.getExpr());
dimension.setName(tagResp.getBizName());
dimension.setOwners("");
dimension.setBizName(tagResp.getBizName());
if (Objects.isNull(dimension.getDataType())) {
dimension.setDataType(DataType.UNKNOWN);
}
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
dimensions.add(dimension);
}
}
if (semanticModel.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().get(entry.getKey()).stream()
.filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> {
dimensions.add(d);
});
}
semanticModel.getDimensionMap().put(entry.getKey(), dimensions);
}
}
// metric ignored
semanticModel.setMetrics(new ArrayList<>());
return semanticModel;
}
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) { public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
return getMetricsByMetricYamlTpl(t); return getMetricsByMetricYamlTpl(t);
} }

View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
@@ -58,6 +59,14 @@ public class QueryController {
return queryService.queryByReq(querySqlReq, user); return queryService.queryByReq(querySqlReq, user);
} }
@PostMapping("/tag")
public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByReq(queryTagReq, user);
}
@PostMapping("/queryMetricDataById") @PostMapping("/queryMetricDataById")
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq, public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
HttpServletRequest request) throws Exception { HttpServletRequest request) throws Exception {

View File

@@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
@@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@@ -83,6 +85,7 @@ public class QueryServiceImpl implements QueryService {
private StatUtils statUtils; private StatUtils statUtils;
private final QueryUtils queryUtils; private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter; private final QueryReqConverter queryReqConverter;
private final TagReqConverter tagReqConverter;
private final Catalog catalog; private final Catalog catalog;
private final AppService appService; private final AppService appService;
private final QueryCache queryCache; private final QueryCache queryCache;
@@ -102,7 +105,7 @@ public class QueryServiceImpl implements QueryService {
StatUtils statUtils, StatUtils statUtils,
QueryUtils queryUtils, QueryUtils queryUtils,
QueryReqConverter queryReqConverter, QueryReqConverter queryReqConverter,
Catalog catalog, TagReqConverter tagReqConverter, Catalog catalog,
AppService appService, AppService appService,
QueryCache queryCache, QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager, SemanticSchemaManager semanticSchemaManager,
@@ -114,6 +117,7 @@ public class QueryServiceImpl implements QueryService {
this.statUtils = statUtils; this.statUtils = statUtils;
this.queryUtils = queryUtils; this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter; this.queryReqConverter = queryReqConverter;
this.tagReqConverter = tagReqConverter;
this.catalog = catalog; this.catalog = catalog;
this.appService = appService; this.appService = appService;
this.queryCache = queryCache; this.queryCache = queryCache;
@@ -185,6 +189,9 @@ public class QueryServiceImpl implements QueryService {
if (semanticQueryReq instanceof QueryMultiStructReq) { if (semanticQueryReq instanceof QueryMultiStructReq) {
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
} }
if (semanticQueryReq instanceof QueryTagReq) {
return buildTagQueryStatement((QueryTagReq) semanticQueryReq);
}
return null; return null;
} }
@@ -220,6 +227,21 @@ public class QueryServiceImpl implements QueryService {
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
} }
private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq)
throws Exception {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq);
schemaFilterReq.setModelIds(queryTagReq.getModelIds());
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp);
queryStatement.setModelIds(queryTagReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setViewId(semanticQueryReq.getViewId()); schemaFilterReq.setViewId(semanticQueryReq.getViewId());

View File

@@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService {
private final DomainService domainService; private final DomainService domainService;
private final ViewService viewService; private final ViewService viewService;
private final ModelRelaService modelRelaService; private final ModelRelaService modelRelaService;
private final TagService tagService;
public SchemaServiceImpl(ModelService modelService, public SchemaServiceImpl(ModelService modelService,
DimensionService dimensionService, DimensionService dimensionService,
MetricService metricService, MetricService metricService,
DomainService domainService, DomainService domainService,
ViewService viewService, ViewService viewService,
ModelRelaService modelRelaService, ModelRelaService modelRelaService,
StatUtils statUtils) { StatUtils statUtils, TagService tagService) {
this.modelService = modelService; this.modelService = modelService;
this.dimensionService = dimensionService; this.dimensionService = dimensionService;
this.metricService = metricService; this.metricService = metricService;
@@ -93,6 +97,7 @@ public class SchemaServiceImpl implements SchemaService {
this.viewService = viewService; this.viewService = viewService;
this.modelRelaService = modelRelaService; this.modelRelaService = modelRelaService;
this.statUtils = statUtils; this.statUtils = statUtils;
this.tagService = tagService;
} }
@SneakyThrows @SneakyThrows
@@ -301,6 +306,11 @@ public class SchemaServiceImpl implements SchemaService {
.flatMap(Collection::stream).collect(Collectors.toList())); .flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL); semanticSchemaResp.setSchemaType(SchemaType.MODEL);
// add tag info
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(schemaFilterReq.getModelIds());
List<TagResp> tagResps = tagService.query(tagFilter);
semanticSchemaResp.setTags(tagResps);
} }
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));

View File

@@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component @Component
@@ -96,6 +96,48 @@ public class StatUtils {
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
initStructStatInfo(queryStructCmd, facadeUser); initStructStatInfo(queryStructCmd, facadeUser);
} }
if (semanticQueryReq instanceof QueryTagReq) {
initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser);
}
}
public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
String traceId = "";
List<String> dimensions = queryTagReq.getGroups();
List<String> metrics = new ArrayList<>();
queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
String user = getUserName(facadeUser);
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryTagReq.getViewId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQueryStructCmd(queryTagReq.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString()))
.setStartTime(System.currentTimeMillis())
.setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators()))
.setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders()))
.setFilterCols(objectMapper.writeValueAsString(
sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters())))
.setUseResultCache(true)
.setUseSqlCache(true)
.setMetrics(objectMapper.writeValueAsString(metrics))
.setDimensions(objectMapper.writeValueAsString(dimensions))
.setQueryOptMode(QueryOptMode.NONE.name());
if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) {
queryStatInfo.setModelId(queryTagReq.getModelIds().get(0));
}
} catch (JsonProcessingException e) {
e.printStackTrace();
}
StatUtils.set(queryStatInfo);
} }
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {

View File

@@ -0,0 +1,97 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class TagReqConverter {
@Value("${query.sql.limitWrapper:true}")
private Boolean limitWrapper;
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QueryTagReq queryTagReq,
SemanticSchemaResp semanticSchemaResp) throws Exception {
QueryStatement queryStatement = new QueryStatement();
// covert to QueryReqConverter
QueryStructReq queryStructReq = new QueryStructReq();
BeanUtils.copyProperties(queryTagReq, queryStructReq);
if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) {
queryStructReq.setDimensionFilters(queryTagReq.getTagFilters());
}
QuerySqlReq querySqlReq = queryStructReq.convert();
if (Objects.nonNull(querySqlReq)) {
log.info("convert to QuerySqlReq {}", querySqlReq);
String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql());
MetricTable metricTable = new MetricTable();
metricTable.setMetrics(new ArrayList<>());
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
semanticSchemaResp.getModelResps().get(0).getBizName()));
metricTable.setAggOption(AggOption.NATIVE);
List<String> allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql());
metricTable.setDimensions(allFields);
metricTable.setAlias(tableName.toLowerCase());
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
BeanUtils.copyProperties(querySqlReq, result);
result.setTables(tables);
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setViewId(querySqlReq.getViewId());
queryStructReq.setQueryType(QueryType.TAG);
QueryParam queryParam = new QueryParam();
convert(queryTagReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(queryTagReq.getViewId());
queryStatement.setEnableLimitWrapper(limitWrapper);
}
return queryStatement;
}
public void convert(QueryTagReq queryTagReq, QueryParam queryParam) {
BeanUtils.copyProperties(queryTagReq, queryParam);
queryParam.setOrders(queryTagReq.getOrders());
queryParam.setMetrics(queryTagReq.getMetrics());
queryParam.setGroups(queryTagReq.getGroups());
queryParam.setDimensionFilters(queryTagReq.getTagFilters());
queryParam.setQueryType(QueryType.TAG);
}
}