From 40ea6a939606b4e08d540322f2ba16db9aab7f21 Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:41:47 +0800 Subject: [PATCH] (feature)(headless) Add tag query api (#790) --- .../supersonic/common/pojo/Aggregator.java | 2 + .../api/pojo/request/QueryStructReq.java | 12 +-- .../api/pojo/request/QueryTagReq.java | 64 ++++++++++++ .../api/pojo/response/SemanticSchemaResp.java | 1 + .../server/aspect/DimValueAspect.java | 24 +++-- .../server/manager/ModelYamlManager.java | 1 + .../server/manager/SemanticSchemaManager.java | 59 ++++++++++- .../headless/server/rest/QueryController.java | 9 ++ .../server/service/impl/QueryServiceImpl.java | 24 ++++- .../service/impl/SchemaServiceImpl.java | 22 +++-- .../headless/server/utils/StatUtils.java | 54 +++++++++-- .../server/utils/TagReqConverter.java | 97 +++++++++++++++++++ 12 files changed, 337 insertions(+), 32 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java index e535de293..1f2da6ddb 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java @@ -19,6 +19,8 @@ public class Aggregator { private List args; + private String alias; + public Aggregator() { } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 470a2afdf..92a39fbd2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -35,11 +39,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - @Data @Slf4j @@ -207,7 +206,8 @@ public class QueryStructReq extends SemanticQueryReq { } sumFunction.setParameters(new ExpressionList(new Column(columnName))); SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction); - selectExpressionItem.setAlias(new Alias(columnName)); + String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName; + selectExpressionItem.setAlias(new Alias(alias)); selectItems.add(selectExpressionItem); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java new file mode 100644 index 000000000..be1a6f3af --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java @@ -0,0 +1,64 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.Order; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.Data; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + +@Data +@Slf4j +@ToString +public class QueryTagReq extends SemanticQueryReq { + + private List groups = new ArrayList<>(); + private List aggregators = new ArrayList<>(); + private List tagFilters = new ArrayList<>(); + private List orders = new ArrayList<>(); + + private Long limit = 20L; + private Long offset = 0L; + + private String tagFiltersDate; + private DateConf dateInfo; + + @Override + public String toCustomizedString() { + StringBuilder stringBuilder = new StringBuilder("{"); + stringBuilder.append("\"viewId\":") + .append(viewId); + stringBuilder.append("\"modelIds\":") + .append(modelIds); + stringBuilder.append(",\"groups\":") + .append(groups); + stringBuilder.append(",\"aggregators\":") + .append(aggregators); + stringBuilder.append(",\"orders\":") + .append(orders); + stringBuilder.append(",\"tagFilters\":") + .append(tagFilters); + stringBuilder.append(",\"dateInfo\":") + .append(dateInfo); + stringBuilder.append(",\"params\":") + .append(params); + stringBuilder.append(",\"limit\":") + .append(limit); + stringBuilder.append('}'); + return stringBuilder.toString(); + } + + public List getMetrics() { + List metrics = Lists.newArrayList(); + if (!CollectionUtils.isEmpty(this.aggregators)) { + metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList()); + } + return metrics; + } +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 543a071b3..d333e51b4 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -22,6 +22,7 @@ public class SemanticSchemaResp { private SchemaType schemaType; private List metrics = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); + private List tags = Lists.newArrayList(); private List modelRelas = Lists.newArrayList(); private List modelResps = Lists.newArrayList(); private ViewResp viewResp; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index afc48d73d..86be574a1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -29,14 +37,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - @Aspect @Component @Slf4j @@ -63,9 +63,17 @@ public class DimValueAspect { if (queryReq instanceof QuerySqlReq) { return handleSqlDimValue(joinPoint); } + + if (queryReq instanceof QueryTagReq) { + return handleTagValue(joinPoint); + } throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); } + public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable { + return (SemanticQueryResp) joinPoint.proceed(); + } + private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable { Object[] args = joinPoint.getArgs(); QueryStructReq queryStructReq = (QueryStructReq) args[0]; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index b73a758ac..c444f491e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -52,6 +52,7 @@ public class ModelYamlManager { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); } dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); + dataModelYamlTpl.setId(modelResp.getId()); return dataModelYamlTpl; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index d35cabff6..07953c41f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; @@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -44,11 +40,16 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Slf4j @Service public class SemanticSchemaManager { + private final Catalog catalog; public SemanticSchemaManager(Catalog catalog) { @@ -87,6 +88,54 @@ public class SemanticSchemaManager { return semanticModel; } + public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception { + if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) { + throw new Exception("semanticSchemaResp tag is empty"); + } + SemanticModel semanticModel = getSemanticModel(semanticSchemaResp); + //Map> dimensions = new HashMap<>(); + Map> tagMap = new HashMap<>(); + for (TagResp tagResp : semanticSchemaResp.getTags()) { + if (!tagMap.containsKey(tagResp.getModelId())) { + tagMap.put(tagResp.getModelId(), new ArrayList<>()); + } + tagMap.get(tagResp.getModelId()).add(tagResp); + } + if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) { + for (Map.Entry entry : semanticModel.getDatasourceMap().entrySet()) { + List dimensions = new ArrayList<>(); + List tagNames = new ArrayList<>(); + if (tagMap.containsKey(entry.getValue().getId())) { + for (TagResp tagResp : tagMap.get(entry.getValue().getId())) { + tagNames.add(tagResp.getBizName()); + Dimension dimension = Dimension.builder().build(); + dimension.setType(""); + dimension.setExpr(tagResp.getExpr()); + dimension.setName(tagResp.getBizName()); + dimension.setOwners(""); + dimension.setBizName(tagResp.getBizName()); + if (Objects.isNull(dimension.getDataType())) { + dimension.setDataType(DataType.UNKNOWN); + } + DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); + dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams); + dimensions.add(dimension); + } + } + if (semanticModel.getDimensionMap().containsKey(entry.getKey())) { + semanticModel.getDimensionMap().get(entry.getKey()).stream() + .filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> { + dimensions.add(d); + }); + } + semanticModel.getDimensionMap().put(entry.getKey(), dimensions); + } + } + // metric ignored + semanticModel.setMetrics(new ArrayList<>()); + return semanticModel; + } + public static List getMetrics(final List t) { return getMetricsByMetricYamlTpl(t); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index 7c222d4f1..3e81ae062 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; @@ -58,6 +59,14 @@ public class QueryController { return queryService.queryByReq(querySqlReq, user); } + @PostMapping("/tag") + public Object queryByTag(@RequestBody QueryTagReq queryTagReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + return queryService.queryByReq(queryTagReq, user); + } + @PostMapping("/queryMetricDataById") public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq, HttpServletRequest request) throws Exception { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index ba8a3d3fd..e159c98b2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; @@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; +import com.tencent.supersonic.headless.server.utils.TagReqConverter; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -83,6 +85,7 @@ public class QueryServiceImpl implements QueryService { private StatUtils statUtils; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; + private final TagReqConverter tagReqConverter; private final Catalog catalog; private final AppService appService; private final QueryCache queryCache; @@ -102,7 +105,7 @@ public class QueryServiceImpl implements QueryService { StatUtils statUtils, QueryUtils queryUtils, QueryReqConverter queryReqConverter, - Catalog catalog, + TagReqConverter tagReqConverter, Catalog catalog, AppService appService, QueryCache queryCache, SemanticSchemaManager semanticSchemaManager, @@ -114,6 +117,7 @@ public class QueryServiceImpl implements QueryService { this.statUtils = statUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; + this.tagReqConverter = tagReqConverter; this.catalog = catalog; this.appService = appService; this.queryCache = queryCache; @@ -185,6 +189,9 @@ public class QueryServiceImpl implements QueryService { if (semanticQueryReq instanceof QueryMultiStructReq) { return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); } + if (semanticQueryReq instanceof QueryTagReq) { + return buildTagQueryStatement((QueryTagReq) semanticQueryReq); + } return null; } @@ -220,6 +227,21 @@ public class QueryServiceImpl implements QueryService { return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); } + private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq) + throws Exception { + SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); + SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq); + schemaFilterReq.setModelIds(queryTagReq.getModelIds()); + SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); + QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp); + queryStatement.setModelIds(queryTagReq.getModelIds()); + queryStatement.setEnableOptimize(queryUtils.enableOptimize()); + queryStatement.setSemanticSchemaResp(semanticSchemaResp); + SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp); + queryStatement.setSemanticModel(semanticModel); + return queryStatement; + } + private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); schemaFilterReq.setViewId(semanticQueryReq.getViewId()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index a1cf5160f..d900d96c6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.TagService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; @@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService { private final DomainService domainService; private final ViewService viewService; private final ModelRelaService modelRelaService; + private final TagService tagService; public SchemaServiceImpl(ModelService modelService, - DimensionService dimensionService, - MetricService metricService, - DomainService domainService, - ViewService viewService, - ModelRelaService modelRelaService, - StatUtils statUtils) { + DimensionService dimensionService, + MetricService metricService, + DomainService domainService, + ViewService viewService, + ModelRelaService modelRelaService, + StatUtils statUtils, TagService tagService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; @@ -93,6 +97,7 @@ public class SchemaServiceImpl implements SchemaService { this.viewService = viewService; this.modelRelaService = modelRelaService; this.statUtils = statUtils; + this.tagService = tagService; } @SneakyThrows @@ -301,6 +306,11 @@ public class SchemaServiceImpl implements SchemaService { .flatMap(Collection::stream).collect(Collectors.toList())); semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setSchemaType(SchemaType.MODEL); + // add tag info + TagFilter tagFilter = new TagFilter(); + tagFilter.setModelIds(schemaFilterReq.getModelIds()); + List tagResps = tagService.query(tagFilter); + semanticSchemaResp.setTags(tagResps); } if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 8f65682b6..25399c801 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.service.ModelService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Component @@ -96,6 +96,48 @@ public class StatUtils { QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); initStructStatInfo(queryStructCmd, facadeUser); } + if (semanticQueryReq instanceof QueryTagReq) { + initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser); + } + } + + public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) { + QueryStat queryStatInfo = new QueryStat(); + String traceId = ""; + List dimensions = queryTagReq.getGroups(); + + List metrics = new ArrayList<>(); + queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); + String user = getUserName(facadeUser); + + try { + queryStatInfo.setTraceId(traceId) + .setViewId(queryTagReq.getViewId()) + .setUser(user) + .setQueryType(QueryType.STRUCT.getValue()) + .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) + .setQueryStructCmd(queryTagReq.toString()) + .setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString())) + .setStartTime(System.currentTimeMillis()) + .setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators())) + .setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups())) + .setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators())) + .setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders())) + .setFilterCols(objectMapper.writeValueAsString( + sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters()))) + .setUseResultCache(true) + .setUseSqlCache(true) + .setMetrics(objectMapper.writeValueAsString(metrics)) + .setDimensions(objectMapper.writeValueAsString(dimensions)) + .setQueryOptMode(QueryOptMode.NONE.name()); + if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) { + queryStatInfo.setModelId(queryTagReq.getModelIds().get(0)); + } + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + StatUtils.set(queryStatInfo); + } public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java new file mode 100644 index 000000000..6fb8070a7 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java @@ -0,0 +1,97 @@ +package com.tencent.supersonic.headless.server.utils; + +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.MetricTable; +import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class TagReqConverter { + + @Value("${query.sql.limitWrapper:true}") + private Boolean limitWrapper; + + @Autowired + private QueryStructUtils queryStructUtils; + + @Autowired + private SqlGenerateUtils sqlGenerateUtils; + + public QueryStatement convert(QueryTagReq queryTagReq, + SemanticSchemaResp semanticSchemaResp) throws Exception { + QueryStatement queryStatement = new QueryStatement(); + // covert to QueryReqConverter + QueryStructReq queryStructReq = new QueryStructReq(); + BeanUtils.copyProperties(queryTagReq, queryStructReq); + if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) { + queryStructReq.setDimensionFilters(queryTagReq.getTagFilters()); + } + QuerySqlReq querySqlReq = queryStructReq.convert(); + if (Objects.nonNull(querySqlReq)) { + log.info("convert to QuerySqlReq {}", querySqlReq); + String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql()); + MetricTable metricTable = new MetricTable(); + metricTable.setMetrics(new ArrayList<>()); + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + semanticSchemaResp.getModelResps().get(0).getBizName())); + metricTable.setAggOption(AggOption.NATIVE); + List allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql()); + metricTable.setDimensions(allFields); + metricTable.setAlias(tableName.toLowerCase()); + List tables = new ArrayList<>(); + tables.add(metricTable); + //.build ParseSqlReq + ViewQueryParam result = new ViewQueryParam(); + BeanUtils.copyProperties(querySqlReq, result); + result.setTables(tables); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), + database.getVersion())) { + result.setSupportWith(false); + result.setWithAlias(false); + } + //.physicalSql by ParseSqlReq + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); + queryStructReq.setViewId(querySqlReq.getViewId()); + queryStructReq.setQueryType(QueryType.TAG); + QueryParam queryParam = new QueryParam(); + convert(queryTagReq, queryParam); + queryStatement.setQueryParam(queryParam); + queryStatement.setViewQueryParam(result); + queryStatement.setIsS2SQL(true); + queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); + queryStatement.setViewId(queryTagReq.getViewId()); + queryStatement.setEnableLimitWrapper(limitWrapper); + } + return queryStatement; + } + + public void convert(QueryTagReq queryTagReq, QueryParam queryParam) { + BeanUtils.copyProperties(queryTagReq, queryParam); + queryParam.setOrders(queryTagReq.getOrders()); + queryParam.setMetrics(queryTagReq.getMetrics()); + queryParam.setGroups(queryTagReq.getGroups()); + queryParam.setDimensionFilters(queryTagReq.getTagFilters()); + queryParam.setQueryType(QueryType.TAG); + } +}