From ed54d7bae3ffaa306c97ef908f7a4aeae9af06d9 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Tue, 9 Jan 2024 10:26:10 +0800 Subject: [PATCH] (improvement)(headless) Query metric/dimension by id in SchemaController and update api header check (#608) * (improvement)(headless) Query metric/dimension by id in SchemaController and update api header check * (improvement)(headless) Fix the issue where metrics were not recognized when the result field contained an aggregate function. --------- Co-authored-by: jolunoluo --- .../config/AuthenticationConfig.java | 9 +++ .../api/authentication/utils/UserHolder.java | 14 +++- .../AuthenticationInterceptor.java | 14 ++-- .../DefaultAuthenticationInterceptor.java | 11 +-- .../headless/api/request/QueryItemReq.java | 13 +++- .../api/request/SchemaItemQueryReq.java | 19 +++++ .../headless/api/response/DimensionResp.java | 2 - .../headless/api/response/MetricResp.java | 8 --- .../parser/calcite/s2sql/HeadlessModel.java | 2 + .../converter/DefaultDimValueConverter.java | 55 +++++++++++++++ .../headless/core/utils/ComponentFactory.java | 6 +- .../headless/server/rest/QueryController.java | 7 +- .../server/rest/SchemaController.java | 6 ++ .../headless/server/service/Catalog.java | 7 +- .../headless/server/service/QueryService.java | 5 +- .../server/service/SchemaService.java | 3 + .../server/service/impl/CatalogImpl.java | 9 +++ .../service/impl/HeadlessQueryEngineImpl.java | 16 +++-- .../server/service/impl/QueryServiceImpl.java | 69 ++++++++++--------- .../service/impl/SchemaServiceImpl.java | 17 ++++- .../server/utils/DimensionConverter.java | 2 + .../server/utils/MetricConverter.java | 2 + .../headless/server/utils/QueryUtils.java | 11 ++- 23 files changed, 234 insertions(+), 73 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SchemaItemQueryReq.java create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java index b7cd770d2..42f3eb109 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/config/AuthenticationConfig.java @@ -24,4 +24,13 @@ public class AuthenticationConfig { @Value("${authentication.token.http.header.key:Authorization}") private String tokenHttpHeaderKey; + @Value("${authentication.app.appId:appId}") + private String appId; + + @Value("${authentication.app.timestamp:timestamp}") + private String timestamp; + + @Value("${authentication.app.signature:signature}") + private String signature; + } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java index eb664ebf2..edb51159e 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java @@ -2,6 +2,11 @@ package com.tencent.supersonic.auth.api.authentication.utils; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; +import com.tencent.supersonic.common.pojo.SysParameter; +import com.tencent.supersonic.common.service.SysParameterService; +import com.tencent.supersonic.common.util.ContextUtils; +import org.springframework.util.CollectionUtils; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -14,7 +19,14 @@ public final class UserHolder { } public static User findUser(HttpServletRequest request, HttpServletResponse response) { - return REPO.findUser(request, response); + User user = REPO.findUser(request, response); + SysParameterService sysParameterService = ContextUtils.getBean(SysParameterService.class); + SysParameter sysParameter = sysParameterService.getSysParameter(); + if (!CollectionUtils.isEmpty(sysParameter.getAdmins()) + && sysParameter.getAdmins().contains(user.getName())) { + user.setIsAdmin(1); + } + return user; } } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java index 119868254..dc9a72f18 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java @@ -5,16 +5,17 @@ import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; import com.tencent.supersonic.auth.authentication.service.UserServiceImpl; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; import com.tencent.supersonic.common.util.S2ThreadContext; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import javax.servlet.http.HttpServletRequest; import org.apache.catalina.connector.RequestFacade; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.apache.tomcat.util.http.MimeHeaders; import org.springframework.util.CollectionUtils; import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; import org.springframework.web.servlet.HandlerInterceptor; +import javax.servlet.http.HttpServletRequest; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; public abstract class AuthenticationInterceptor implements HandlerInterceptor { @@ -58,6 +59,11 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor { return "true".equalsIgnoreCase(internal); } + protected boolean isAppRequest(HttpServletRequest request) { + String appId = request.getHeader(authenticationConfig.getAppId()); + return StringUtils.isNotBlank(appId); + } + protected void reflectSetparam(HttpServletRequest request, String key, String value) { try { if (request instanceof StandardMultipartHttpServletRequest) { diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java index b62f72072..f38f3b4ec 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java @@ -10,12 +10,12 @@ import com.tencent.supersonic.common.pojo.exception.AccessException; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.ThreadContext; -import java.lang.reflect.Method; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.web.method.HandlerMethod; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.lang.reflect.Method; @Slf4j public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor { @@ -35,7 +35,10 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor setFakerUser(request); return true; } - + if (isAppRequest(request)) { + setFakerUser(request); + return true; + } if (handler instanceof HandlerMethod) { HandlerMethod handlerMethod = (HandlerMethod) handler; Method method = handlerMethod.getMethod(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryItemReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryItemReq.java index 939da96de..a5e573ad6 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryItemReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryItemReq.java @@ -3,18 +3,25 @@ package com.tencent.supersonic.headless.api.request; import com.tencent.supersonic.common.pojo.DateConf; import lombok.Data; -import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Size; import java.util.List; @Data public class QueryItemReq { - @NotEmpty(message = "ids不可为空") + @Size(max = 5, min = 1) private List ids; private DateConf dateConf = new DateConf(); //result size of single id - private Long limit; + private Long limit = 1000L; + + public Long getLimit() { + if (limit > 10000) { + return 10000L; + } + return limit; + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SchemaItemQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SchemaItemQueryReq.java new file mode 100644 index 000000000..f4b2ed352 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SchemaItemQueryReq.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.headless.api.request; + +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import lombok.Data; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +@Data +public class SchemaItemQueryReq { + + @NotEmpty(message = "id个数不可为空") + private List ids; + + //METRIC, DIMENSION + @NotNull(message = "type不可为空") + private TypeEnums type; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/DimensionResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/DimensionResp.java index 1ade9e5e8..692eef73b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/DimensionResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/DimensionResp.java @@ -20,8 +20,6 @@ public class DimensionResp extends SchemaItem { private String expr; - private String fullPath; - private String modelName; private String modelBizName; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java index 7ceb1d843..23fdec369 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java @@ -5,7 +5,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; @@ -78,13 +77,6 @@ public class MetricResp extends SchemaItem { .collect(Collectors.joining(",")); } - public List getMeasures() { - if (typeParams == null) { - return Lists.newArrayList(); - } - return typeParams.getMeasures(); - } - public String getDefaultAgg() { return typeParams.getMeasures().get(0).getAgg(); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java index 31aec10cd..e86428d1b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.api.response.DatabaseResp; +import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import lombok.Data; import java.util.ArrayList; @@ -22,6 +23,7 @@ public class HeadlessModel { private List joinRelations; private ItemDateResp dataDate; private DatabaseResp databaseResp; + private List modelSchemaResps; public List getDimensions() { return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java new file mode 100644 index 000000000..3a5afd1d5 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java @@ -0,0 +1,55 @@ +package com.tencent.supersonic.headless.core.parser.converter; + +import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.DimensionResp; +import com.tencent.supersonic.headless.core.parser.HeadlessConverter; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +@Slf4j +@Component("DefaultDimValueConverter") +public class DefaultDimValueConverter implements HeadlessConverter { + + @Override + public boolean accept(QueryStatement queryStatement) { + if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { + return false; + } + return true; + } + + @Override + public void convert(QueryStatement queryStatement) { + QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); + List dimensionResps = queryStatement.getHeadlessModel().getModelSchemaResps() + .stream().flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream()) + .filter(dimSchemaResp -> !CollectionUtils.isEmpty(dimSchemaResp.getDefaultValues())) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(dimensionResps)) { + return; + } + log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd); + //add dimension default value to filter + List dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream() + .map(Filter::getBizName).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(dimensionFilterBizName)) { + return; + } + for (DimensionResp dimensionResp : dimensionResps) { + Filter filter = new Filter(); + filter.setBizName(dimensionResp.getBizName()); + filter.setValue(dimensionResp.getDefaultValues()); + filter.setOperator(FilterOperatorEnum.IN); + filter.setName(dimensionResp.getName()); + queryStructCmd.getDimensionFilters().add(filter); + } + } + +} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 419ece0c7..3768030c1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -3,12 +3,13 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.executor.JdbcExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.core.optimizer.DetailQuery; +import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser; -import com.tencent.supersonic.headless.core.optimizer.DetailQuery; -import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter; +import com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter; import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter; import java.util.ArrayList; import java.util.HashMap; @@ -81,6 +82,7 @@ public class ComponentFactory { } private static void initSemanticConverter() { + headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index 2177f2782..ef6533b0a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -33,6 +33,7 @@ import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import java.util.List; @RestController @@ -66,10 +67,10 @@ public class QueryController { return queryService.queryByStructWithAuth(queryStructReq, user); } - @PostMapping("/metricDataQueryById") - public ItemQueryResultResp metricDataQueryById(@RequestBody QueryItemReq queryApiReq, + @PostMapping("/queryMetricDataById") + public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq, HttpServletRequest request) throws Exception { - return queryService.metricDataQueryById(queryApiReq, request); + return queryService.queryMetricDataById(queryApiReq, request); } @PostMapping("/download/struct") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java index 08697d258..6cb6753bd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; +import com.tencent.supersonic.headless.api.request.SchemaItemQueryReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.MetricResp; @@ -72,4 +73,9 @@ public class SchemaController { return schemaService.queryMetric(pageMetricCmd, user); } + @PostMapping("/schemaItem/list") + public List querySchemaItem(@RequestBody SchemaItemQueryReq schemaItemQueryReq) { + return schemaService.querySchemaItem(schemaItemQueryReq); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java index df67d0b78..c7d8b7af2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; +import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; @@ -34,9 +35,11 @@ public interface Catalog { List getModelRela(List modelIds); + List getModelSchema(List modelIds); + void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, - List dataModelYamlTplList, List metricYamlTplList, - Map modelIdName); + List dataModelYamlTplList, List metricYamlTplList, + Map modelIdName); ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index 8475aaf77..257bc4cc4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ItemUseReq; -import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; @@ -39,9 +38,7 @@ public interface QueryService { ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; @ApiHeaderCheck - ItemQueryResultResp metricDataQueryById(QueryItemReq queryApiReq, + ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception; - QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 668ea3da1..bc52a6e40 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; +import com.tencent.supersonic.headless.api.request.SchemaItemQueryReq; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.DimensionResp; @@ -22,6 +23,8 @@ public interface SchemaService { PageInfo queryMetric(PageMetricReq pageMetricReq, User user); + List querySchemaItem(SchemaItemQueryReq schemaItemQueryReq); + List getDomainList(User user); List getModelList(User user, AuthType authType, Long domainId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java index 066c0a545..da0823a99 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java @@ -3,10 +3,12 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; +import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; +import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; @@ -103,6 +105,13 @@ public class CatalogImpl implements Catalog { return modelRespList; } + @Override + public List getModelSchema(List modelIds) { + ModelSchemaFilterReq modelSchemaFilterReq = new ModelSchemaFilterReq(); + modelSchemaFilterReq.setModelIds(modelIds); + return modelService.fetchModelSchema(modelSchemaFilterReq); + } + @Override public void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, List dataModelYamlTplList, List metricYamlTplList, diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java index 5c985263b..4f14be034 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.QueryParser; @@ -19,6 +20,7 @@ import com.tencent.supersonic.headless.server.utils.QueryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.List; @Slf4j @Component @@ -55,10 +57,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { public QueryStatement plan(QueryStatement queryStatement) throws Exception { queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - HeadlessModel headlessModel = headlessSchemaManager.get(queryStatement.getQueryStructReq().getModelIdStr()); - ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStatement.getQueryStructReq()); - headlessModel.setDataDate(itemDateResp); - queryStatement.setHeadlessModel(headlessModel); + queryStatement.setHeadlessModel(getHeadLessModel(queryStatement)); queryStatement = queryParser.logicSql(queryStatement); queryUtils.checkSqlParse(queryStatement); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); @@ -99,4 +98,13 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { return queryParser.parser(queryStatement); } + private HeadlessModel getHeadLessModel(QueryStatement queryStatement) throws Exception { + HeadlessModel headlessModel = headlessSchemaManager.get(queryStatement.getQueryStructReq().getModelIdStr()); + ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStatement.getQueryStructReq()); + headlessModel.setDataDate(itemDateResp); + List modelSchemaResps = catalog.getModelSchema(queryStatement.getModelIds()); + headlessModel.setModelSchemaResps(modelSchemaResps); + return headlessModel; + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 099d88bf7..d77dd67ec 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -39,7 +39,6 @@ import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck; import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission; import com.tencent.supersonic.headless.server.annotation.StructDataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; @@ -52,6 +51,13 @@ import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -59,13 +65,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import javax.servlet.http.HttpServletRequest; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; @Service @@ -136,17 +135,17 @@ public class QueryServiceImpl implements QueryService { } @Override - public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception { + public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception { QueryResultWithSchemaResp queryResultWithColumns = null; - log.info("[queryStructCmd:{}]", queryStructCmd); + log.info("[queryStructReq:{}]", queryStructReq); try { - statUtils.initStatInfo(queryStructCmd, user); - String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructCmd.getModelIds()), - queryStructCmd.generateCommandMd5()); - handleGlobalCacheDisable(queryStructCmd); - boolean isCache = isCache(queryStructCmd); + statUtils.initStatInfo(queryStructReq, user); + String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructReq.getModelIds()), + queryStructReq.generateCommandMd5()); + handleGlobalCacheDisable(queryStructReq); + boolean isCache = isCache(queryStructReq); if (isCache) { - queryResultWithColumns = queryByCache(cacheKey, queryStructCmd); + queryResultWithColumns = queryByCache(cacheKey, queryStructReq); if (queryResultWithColumns != null) { statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); return queryResultWithColumns; @@ -154,7 +153,7 @@ public class QueryServiceImpl implements QueryService { } StatUtils.get().setUseResultCache(false); QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructCmd); + queryStatement.setQueryStructReq(queryStructReq); queryStatement.setIsS2SQL(false); queryStatement = headlessQueryEngine.plan(queryStatement); QueryExecutor queryExecutor = headlessQueryEngine.route(queryStatement); @@ -177,8 +176,8 @@ public class QueryServiceImpl implements QueryService { @Override @StructDataPermission @SneakyThrows - public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) { - return queryByStruct(queryStructCmd, user); + public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) { + return queryByStruct(queryStructReq, user); } @Override @@ -215,9 +214,9 @@ public class QueryServiceImpl implements QueryService { private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception { List sqlParsers = new ArrayList<>(); - for (QueryStructReq queryStructCmd : queryMultiStructReq.getQueryStructReqs()) { + for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructCmd); + queryStatement.setQueryStructReq(queryStructReq); queryStatement.setIsS2SQL(false); queryStatement = headlessQueryEngine.plan(queryStatement); queryUtils.checkSqlParse(queryStatement); @@ -234,11 +233,11 @@ public class QueryServiceImpl implements QueryService { return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user); } - private void handleGlobalCacheDisable(QueryStructReq queryStructCmd) { + private void handleGlobalCacheDisable(QueryStructReq queryStructReq) { if (!cacheEnable) { Cache cacheInfo = new Cache(); cacheInfo.setCache(false); - queryStructCmd.setCacheInfo(cacheInfo); + queryStructReq.setCacheInfo(cacheInfo); } } @@ -281,8 +280,7 @@ public class QueryServiceImpl implements QueryService { } @Override - @ApiHeaderCheck - public ItemQueryResultResp metricDataQueryById(QueryItemReq queryItemReq, + public ItemQueryResultResp queryMetricDataById(QueryItemReq queryItemReq, HttpServletRequest request) throws Exception { AppDetailResp appDetailResp = getAppDetailResp(request); authCheck(appDetailResp, queryItemReq.getIds(), ApiItemType.METRIC); @@ -300,6 +298,9 @@ public class QueryServiceImpl implements QueryService { private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception { MetricResp metricResp = catalog.getMetric(item.getId()); + item.setCreatedBy(metricResp.getCreatedBy()); + item.setBizName(metricResp.getBizName()); + item.setName(metricResp.getName()); List items = item.getRelateItems(); List dimensionResps = Lists.newArrayList(); if (!org.springframework.util.CollectionUtils.isEmpty(items)) { @@ -366,27 +367,27 @@ public class QueryServiceImpl implements QueryService { } public QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception { - QueryStructReq queryStructCmd = new QueryStructReq(); - return headlessQueryEngine.physicalSql(queryStructCmd, metricReq); + QueryStructReq queryStructReq = new QueryStructReq(); + return headlessQueryEngine.physicalSql(queryStructReq, metricReq); } - private boolean isCache(QueryStructReq queryStructCmd) { + private boolean isCache(QueryStructReq queryStructReq) { if (!cacheEnable) { return false; } - if (queryStructCmd.getCacheInfo() != null) { - return queryStructCmd.getCacheInfo().getCache(); + if (queryStructReq.getCacheInfo() != null) { + return queryStructReq.getCacheInfo().getCache(); } return false; } - private boolean isCache(QueryMultiStructReq queryStructCmd) { + private boolean isCache(QueryMultiStructReq queryStructReq) { if (!cacheEnable) { return false; } - if (!CollectionUtils.isEmpty(queryStructCmd.getQueryStructReqs()) - && queryStructCmd.getQueryStructReqs().get(0).getCacheInfo() != null) { - return queryStructCmd.getQueryStructReqs().get(0).getCacheInfo().getCache(); + if (!CollectionUtils.isEmpty(queryStructReq.getQueryStructReqs()) + && queryStructReq.getQueryStructReqs().get(0).getCacheInfo() != null) { + return queryStructReq.getQueryStructReqs().get(0).getCacheInfo().getCache(); } return false; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index df4051fd8..08d84f86a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -4,18 +4,21 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.request.ItemUseReq; -import com.tencent.supersonic.headless.api.response.ItemUseResp; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; +import com.tencent.supersonic.headless.api.request.SchemaItemQueryReq; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; +import com.tencent.supersonic.headless.api.response.ItemUseResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -123,6 +126,18 @@ public class SchemaServiceImpl implements SchemaService { return metricService.queryMetric(pageMetricReq, user); } + @Override + public List querySchemaItem(SchemaItemQueryReq schemaItemQueryReq) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(schemaItemQueryReq.getIds()); + if (TypeEnums.METRIC.equals(schemaItemQueryReq.getType())) { + return metricService.getMetrics(metaFilter); + } else if (TypeEnums.DIMENSION.equals(schemaItemQueryReq.getType())) { + return dimensionService.getDimensions(metaFilter); + } + throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().getName()); + } + @Override public List getDomainList(User user) { return domainService.getDomainListWithAdminAuth(user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index e1462c6f7..3491dff44 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DimValueMap; @@ -72,6 +73,7 @@ public class DimensionConverter { if (Strings.isNotEmpty(dimensionDO.getDataType())) { dimensionResp.setDataType(DataTypeEnums.of(dimensionDO.getDataType())); } + dimensionResp.setTypeEnum(TypeEnums.DIMENSION); return dimensionResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 37747094d..79c874996 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.RelateDimension; @@ -65,6 +66,7 @@ public class MetricConverter { if (metricDO.getExt() != null) { metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class)); } + metricResp.setTypeEnum(TypeEnums.METRIC); return metricResp; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 9e3e77c44..6e47dd672 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -93,7 +93,7 @@ public class QueryUtils { }); List columns = queryResultWithColumns.getColumns(); columns.forEach(column -> { - String nameEn = column.getNameEn().toLowerCase(); + String nameEn = getName(column.getNameEn().toLowerCase()); if (nameEn.contains(JOIN_UNDERLINE)) { nameEn = nameEn.split(JOIN_UNDERLINE)[1]; } @@ -164,6 +164,15 @@ public class QueryUtils { }); } + private String getName(String nameEn) { + Pattern pattern = Pattern.compile("\\((.*?)\\)"); + Matcher matcher = pattern.matcher(nameEn); + if (matcher.find()) { + return matcher.group(1).replaceAll("`", ""); + } + return nameEn; + } + private String getNameEnByRegex(String nameEn, String pattern) { Pattern p = Pattern.compile(pattern); Matcher m = p.matcher(nameEn);