From af53812d089d30362596e07d3280bb281d2b5cd8 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:29:32 +0800 Subject: [PATCH] (improvement)(Headless) remove queryType in dataSet (#830) --- .../headless/api/pojo/DataSetModelConfig.java | 5 +- .../api/pojo/response/DataSetResp.java | 5 - .../server/aspect/DimValueAspect.java | 11 +- .../server/rest/MetricController.java | 38 ++--- .../rest/api/TagQueryApiController.java | 11 +- .../service/impl/DataSetServiceImpl.java | 53 ++----- .../server/service/impl/QueryServiceImpl.java | 32 +--- .../service/impl/SchemaServiceImpl.java | 19 +-- .../headless/server/utils/TagConverter.java | 140 ------------------ .../supersonic/BenchMarkDemoDataLoader.java | 9 +- .../supersonic/DuSQLDemoDataLoader.java | 12 +- .../supersonic/ModelDemoDataLoader.java | 9 +- 12 files changed, 59 insertions(+), 285 deletions(-) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagConverter.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java index dd1fcd9d4..ceb4e4b06 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetModelConfig.java @@ -20,12 +20,9 @@ public class DataSetModelConfig { private List dimensions = Lists.newArrayList(); - private List tagIds = Lists.newArrayList(); - - public DataSetModelConfig(Long id, List dimensions, List metrics, List tagIds) { + public DataSetModelConfig(Long id, List dimensions, List metrics) { this.id = id; this.metrics = metrics; this.dimensions = dimensions; - this.tagIds = tagIds; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java index 67a5dc29a..502db5357 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java @@ -47,11 +47,6 @@ public class DataSetResp extends SchemaItem { .collect(Collectors.toList()); } - public List getAllTags() { - return getDataSetModelConfigs().stream().map(DataSetModelConfig::getTagIds) - .flatMap(Collection::stream).collect(Collectors.toList()); - } - public List getAllIncludeAllModels() { return getDataSetModelConfigs().stream().filter(DataSetModelConfig::isIncludesAll) .map(DataSetModelConfig::getId) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index be3a70701..210c3a0d9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -13,7 +13,6 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; @@ -47,7 +46,7 @@ public class DimValueAspect { @Autowired private DimensionService dimensionService; - @Around("execution(* com.tencent.supersonic.headless.server.service.ChatQueryService.queryByReq(..))") + @Around("execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByReq(..))") public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapEnable) { log.debug("dimensionValueMapEnable is false, skip dimensionValueMap"); @@ -63,17 +62,9 @@ public class DimValueAspect { if (queryReq instanceof QuerySqlReq) { return handleSqlDimValue(joinPoint); } - - if (queryReq instanceof QueryTagReq) { - return handleTagValue(joinPoint); - } throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); } - public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable { - return (SemanticQueryResp) joinPoint.proceed(); - } - private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable { Object[] args = joinPoint.getArgs(); QueryStructReq queryStructReq = (QueryStructReq) args[0]; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java index 5125a38cf..e6b43cef4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java @@ -42,24 +42,24 @@ public class MetricController { @PostMapping("/createMetric") public MetricResp createMetric(@RequestBody MetricReq metricReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletRequest request, + HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); return metricService.createMetric(metricReq, user); } @PostMapping("/updateMetric") public MetricResp updateMetric(@RequestBody MetricReq metricReq, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletRequest request, + HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); return metricService.updateMetric(metricReq, user); } @PostMapping("/batchUpdateStatus") public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.batchUpdateStatus(metaBatchReq, user); return true; @@ -67,8 +67,8 @@ public class MetricController { @PostMapping("/mockMetricAlias") public List mockMetricAlias(@RequestBody MetricBaseReq metricReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.mockAlias(metricReq, "indicator", user); } @@ -86,8 +86,8 @@ public class MetricController { @PostMapping("/queryMetric") public PageInfo queryMetric(@RequestBody PageMetricReq pageMetricReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.queryMetric(pageMetricReq, user); } @@ -95,22 +95,22 @@ public class MetricController { @Deprecated @GetMapping("getMetric/{modelId}/{bizName}") public MetricResp getMetric(@PathVariable("modelId") Long modelId, - @PathVariable("bizName") String bizName) { + @PathVariable("bizName") String bizName) { return metricService.getMetric(modelId, bizName); } @GetMapping("getMetric/{id}") public MetricResp getMetric(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.getMetric(id, user); } @DeleteMapping("deleteMetric/{id}") public Boolean deleteMetric(@PathVariable("id") Long id, - HttpServletRequest request, - HttpServletResponse response) throws Exception { + HttpServletRequest request, + HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); metricService.deleteMetric(id, user); return true; @@ -135,8 +135,8 @@ public class MetricController { @PostMapping("/saveMetricQueryDefaultConfig") public boolean saveMetricQueryDefaultConfig(@RequestBody MetricQueryDefaultConfig queryDefaultConfig, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); metricService.saveMetricQueryDefaultConfig(queryDefaultConfig, user); return true; @@ -144,8 +144,8 @@ public class MetricController { @RequestMapping("getMetricQueryDefaultConfig/{metricId}") public MetricQueryDefaultConfig getMetricQueryDefaultConfig(@PathVariable("metricId") Long metricId, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return metricService.getMetricQueryDefaultConfig(metricId, user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java index f3584262b..8e943edb1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java @@ -2,8 +2,10 @@ package com.tencent.supersonic.headless.server.rest.api; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.server.service.QueryService; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -11,9 +13,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - @RestController @RequestMapping("/api/semantic/query") @Slf4j @@ -23,11 +22,11 @@ public class TagQueryApiController { private QueryService queryService; @PostMapping("/tag") - public Object queryByTag(@RequestBody QueryTagReq queryTagReq, + public Object queryByTag(@RequestBody QueryStructReq queryStructReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - return queryService.queryByReq(queryTagReq, user); + return queryService.queryByReq(queryStructReq.convert(), user); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java index 0c950aafc..832ad0b84 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java @@ -8,39 +8,28 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; -import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; +import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO; import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.DataSetService; -import java.util.Objects; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.Arrays; import java.util.Comparator; import java.util.Date; @@ -51,6 +40,13 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service public class DataSetServiceImpl @@ -171,11 +167,6 @@ public class DataSetServiceImpl dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(","))); dataSetResp.setTypeEnum(TypeEnums.DATASET); - String queryType = dataSetDO.getQueryType(); - if (Objects.isNull(queryType)) { - queryType = QueryType.METRIC.name(); - } - dataSetResp.setQueryType(QueryType.valueOf(queryType)); return dataSetResp; } @@ -184,8 +175,6 @@ public class DataSetServiceImpl BeanMapper.mapper(dataSetReq, dataSetDO); dataSetDO.setDataSetDetail(JSONObject.toJSONString(dataSetReq.getDataSetDetail())); dataSetDO.setQueryConfig(JSONObject.toJSONString(dataSetReq.getQueryConfig())); - QueryType queryType = getQueryType(dataSetReq); - dataSetDO.setQueryType(queryType.name()); return dataSetDO; } @@ -198,24 +187,6 @@ public class DataSetServiceImpl return queryReq; } - private QueryType getQueryType(DataSetReq dataSetReq) { - QueryType queryType = dataSetReq.getQueryType(); - if (Objects.nonNull(queryType)) { - return queryType; - } - List dataSetModelConfigs = dataSetReq.getDataSetDetail().getDataSetModelConfigs(); - if (CollectionUtils.isEmpty(dataSetModelConfigs)) { - return QueryType.METRIC; - } - - Set collect = dataSetModelConfigs.stream() - .filter(config -> !CollectionUtils.isEmpty(config.getTagIds())).collect(Collectors.toSet()); - if (CollectionUtils.isEmpty(collect)) { - return QueryType.METRIC; - } - return QueryType.TAG; - } - public static boolean checkAdminPermission(User user, DataSetResp dataSetResp) { List admins = dataSetResp.getAdmins(); if (user.isSuperAdmin()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 3502f6934..734cc7c79 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.ApiItemType; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; @@ -22,7 +21,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; @@ -52,7 +50,6 @@ import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; -import com.tencent.supersonic.headless.server.utils.TagConverter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -73,7 +70,6 @@ public class QueryServiceImpl implements QueryService { private StatUtils statUtils; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; - private final TagConverter tagConverter; private final Catalog catalog; private final AppService appService; private final SemanticSchemaManager semanticSchemaManager; @@ -84,7 +80,7 @@ public class QueryServiceImpl implements QueryService { StatUtils statUtils, QueryUtils queryUtils, QueryReqConverter queryReqConverter, - TagConverter tagConverter, Catalog catalog, + Catalog catalog, AppService appService, SemanticSchemaManager semanticSchemaManager, DefaultQueryParser queryParser, @@ -92,7 +88,6 @@ public class QueryServiceImpl implements QueryService { this.statUtils = statUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; - this.tagConverter = tagConverter; this.catalog = catalog; this.appService = appService; this.semanticSchemaManager = semanticSchemaManager; @@ -160,9 +155,6 @@ public class QueryServiceImpl implements QueryService { if (semanticQueryReq instanceof QueryMultiStructReq) { return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); } - if (semanticQueryReq instanceof QueryTagReq) { - return buildTagQueryStatement((QueryTagReq) semanticQueryReq); - } return null; } @@ -177,12 +169,7 @@ public class QueryServiceImpl implements QueryService { queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setDataSetId(queryStructReq.getDataSetId()); queryStatement.setSemanticSchemaResp(semanticSchemaResp); - if (QueryType.TAG.equals(semanticSchemaResp.getQueryType())) { - queryStatement = tagConverter.convert(queryStructReq, semanticSchemaResp); - queryStatement.setSemanticModel(semanticSchemaManager.getTagSemanticModel(semanticSchemaResp)); - } else { - queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp)); - } + queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp)); return queryStatement; } @@ -202,21 +189,6 @@ public class QueryServiceImpl implements QueryService { return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); } - private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq) - throws Exception { - SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); - SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq); - schemaFilterReq.setModelIds(queryTagReq.getModelIds()); - SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); - QueryStatement queryStatement = tagConverter.convert(queryTagReq, semanticSchemaResp); - queryStatement.setModelIds(queryTagReq.getModelIds()); - queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - queryStatement.setSemanticSchemaResp(semanticSchemaResp); - SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp); - queryStatement.setSemanticModel(semanticModel); - return queryStatement; - } - private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index 549be85c7..f5c014279 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.service.impl; +import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; + import com.github.pagehelper.PageInfo; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -47,13 +49,6 @@ import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.StatUtils; -import com.tencent.supersonic.headless.server.utils.TagConverter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -62,8 +57,11 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Slf4j @Service @@ -181,7 +179,6 @@ public class SchemaServiceImpl implements SchemaService { TagFilter tagFilter = new TagFilter(); tagFilter.setModelIds(modelIds); - List tagRespList = tagService.getTags(tagFilter); List dataSetSchemaResps = new ArrayList<>(); for (Long dataSetId : dataSetRespMap.keySet()) { @@ -200,8 +197,6 @@ public class SchemaServiceImpl implements SchemaService { dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList())); - List tagResps = TagConverter.filterByDataSet(tagRespList, dataSetResp); - dataSetSchemaResp.setTags(tagResps); dataSetSchemaResps.add(dataSetSchemaResp); } fillStaticInfo(dataSetSchemaResps); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagConverter.java deleted file mode 100644 index c6c03318c..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagConverter.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.tencent.supersonic.headless.server.utils; - -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.TagResp; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class TagConverter { - - @Value("${query.sql.limitWrapper:true}") - private Boolean limitWrapper; - - @Autowired - private QueryStructUtils queryStructUtils; - - @Autowired - private SqlGenerateUtils sqlGenerateUtils; - - public QueryStatement convert(QueryStructReq queryStructReq, - SemanticSchemaResp semanticSchemaResp) throws Exception { - QueryStatement queryStatement = new QueryStatement(); - // covert to QueryReqConverter - BeanUtils.copyProperties(queryStructReq.convert(), queryStructReq); - QuerySqlReq querySqlReq = queryStructReq.convert(); - convert(querySqlReq, semanticSchemaResp, queryStatement, queryStructReq); - QueryParam queryParam = new QueryParam(); - convert(queryStructReq, queryParam); - queryStatement.setQueryParam(queryParam); - queryStatement.setDataSetId(queryStructReq.getDataSetId()); - return queryStatement; - - - } - - public void convert(QuerySqlReq querySqlReq, - SemanticSchemaResp semanticSchemaResp, QueryStatement queryStatement, QueryStructReq queryStructReq) - throws Exception { - if (Objects.nonNull(querySqlReq)) { - log.info("convert to QuerySqlReq {}", querySqlReq); - String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql()); - MetricTable metricTable = new MetricTable(); - metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - semanticSchemaResp.getModelResps().get(0).getBizName())); - metricTable.setAggOption(AggOption.NATIVE); - List allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql()); - metricTable.setDimensions(allFields); - metricTable.setAlias(tableName.toLowerCase()); - List tables = new ArrayList<>(); - tables.add(metricTable); - //.build ParseSqlReq - DataSetQueryParam result = new DataSetQueryParam(); - BeanUtils.copyProperties(querySqlReq, result); - result.setTables(tables); - DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), - database.getVersion())) { - result.setSupportWith(false); - result.setWithAlias(false); - } - //.physicalSql by ParseSqlReq - queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); - queryStructReq.setDataSetId(querySqlReq.getDataSetId()); - queryStructReq.setQueryType(QueryType.TAG); - - queryStatement.setDataSetQueryParam(result); - queryStatement.setIsS2SQL(true); - queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - - queryStatement.setEnableLimitWrapper(limitWrapper); - } - } - - public QueryStatement convert(QueryTagReq queryTagReq, - SemanticSchemaResp semanticSchemaResp) throws Exception { - QueryStatement queryStatement = new QueryStatement(); - // covert to QueryReqConverter - QueryStructReq queryStructReq = new QueryStructReq(); - BeanUtils.copyProperties(queryTagReq, queryStructReq); - queryStructReq.setModelIds(queryTagReq.getModelIdSet()); - if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) { - queryStructReq.setDimensionFilters(queryTagReq.getTagFilters()); - } - queryStructReq.setQueryType(QueryType.TAG); - QuerySqlReq querySqlReq = queryStructReq.convert(); - convert(querySqlReq, semanticSchemaResp, queryStatement, queryStructReq); - QueryParam queryParam = new QueryParam(); - convert(queryTagReq, queryParam); - queryStatement.setQueryParam(queryParam); - queryStatement.setDataSetId(queryTagReq.getDataSetId()); - return queryStatement; - } - - public void convert(QueryTagReq queryTagReq, QueryParam queryParam) { - BeanUtils.copyProperties(queryTagReq, queryParam); - queryParam.setOrders(queryTagReq.getOrders()); - queryParam.setMetrics(queryTagReq.getMetrics()); - queryParam.setGroups(queryTagReq.getGroups()); - queryParam.setDimensionFilters(queryTagReq.getTagFilters()); - queryParam.setQueryType(QueryType.TAG); - } - - public void convert(QueryStructReq queryTagReq, QueryParam queryParam) { - BeanUtils.copyProperties(queryTagReq, queryParam); - queryParam.setOrders(queryTagReq.getOrders()); - queryParam.setMetrics(queryTagReq.getMetrics()); - queryParam.setGroups(queryTagReq.getGroups()); - queryParam.setDimensionFilters(queryTagReq.getDimensionFilters()); - queryParam.setQueryType(QueryType.TAG); - } - - public static List filterByDataSet(List tagResps, DataSetResp dataSetResp) { - return tagResps.stream().filter(tagResp -> dataSetResp.getAllTags().contains(tagResp.getId()) - || dataSetResp.getAllIncludeAllModels().contains(tagResp.getModelId())).collect(Collectors.toList()); - } -} diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index 51d492f50..76eb5fa5c 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -221,11 +221,10 @@ public class BenchMarkDemoDataLoader { viewReq.setDescription("包含cspider数据集相关标签和指标信息"); viewReq.setAdmins(Lists.newArrayList("admin")); List viewModelConfigs = Lists.newArrayList( - new DataSetModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList(), Lists.newArrayList()), - new DataSetModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList(), Lists.newArrayList()), - new DataSetModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList(), Lists.newArrayList()), - new DataSetModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L), - Lists.newArrayList()) + new DataSetModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), + new DataSetModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), + new DataSetModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), + new DataSetModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) ); DataSetDetail viewDetail = new DataSetDetail(); viewDetail.setDataSetModelConfigs(viewModelConfigs); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java index f9fb33387..ed13483fa 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/DuSQLDemoDataLoader.java @@ -262,14 +262,10 @@ public class DuSQLDemoDataLoader { viewReq.setDescription("DuSQL互联网企业数据源相关的指标和维度等"); viewReq.setAdmins(Lists.newArrayList("admin")); List viewModelConfigs = Lists.newArrayList( - new DataSetModelConfig(9L, Lists.newArrayList(16L, 17L, 18L, 19L, 20L), Lists.newArrayList(10L, 11L), - Lists.newArrayList()), - new DataSetModelConfig(10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L), - Lists.newArrayList()), - new DataSetModelConfig(11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L), - Lists.newArrayList()), - new DataSetModelConfig(12L, Lists.newArrayList(24L), Lists.newArrayList(16L, 17L, 18L, 19L), - Lists.newArrayList())); + new DataSetModelConfig(9L, Lists.newArrayList(16L, 17L, 18L, 19L, 20L), Lists.newArrayList(10L, 11L)), + new DataSetModelConfig(10L, Lists.newArrayList(21L, 22L, 23L), Lists.newArrayList(12L)), + new DataSetModelConfig(11L, Lists.newArrayList(), Lists.newArrayList(13L, 14L, 15L)), + new DataSetModelConfig(12L, Lists.newArrayList(24L), Lists.newArrayList(16L, 17L, 18L, 19L))); DataSetDetail viewDetail = new DataSetDetail(); viewDetail.setDataSetModelConfigs(viewModelConfigs); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java index b13986380..b4f47ff96 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java @@ -513,9 +513,9 @@ public class ModelDemoDataLoader { dataSetReq.setDescription("包含超音数访问统计相关的指标和维度等"); dataSetReq.setAdmins(Lists.newArrayList("admin")); List dataSetModelConfigs = Lists.newArrayList( - new DataSetModelConfig(1L, Lists.newArrayList(1L, 2L), Lists.newArrayList(), Lists.newArrayList()), - new DataSetModelConfig(2L, Lists.newArrayList(), Lists.newArrayList(1L, 2L, 3L), Lists.newArrayList()), - new DataSetModelConfig(3L, Lists.newArrayList(3L), Lists.newArrayList(4L), Lists.newArrayList())); + new DataSetModelConfig(1L, Lists.newArrayList(1L, 2L), Lists.newArrayList()), + new DataSetModelConfig(2L, Lists.newArrayList(), Lists.newArrayList(1L, 2L, 3L)), + new DataSetModelConfig(3L, Lists.newArrayList(3L), Lists.newArrayList(4L))); DataSetDetail dataSetDetail = new DataSetDetail(); dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs); @@ -540,8 +540,7 @@ public class ModelDemoDataLoader { dataSetReq.setDescription("包含艺人相关标签和指标信息"); dataSetReq.setAdmins(Lists.newArrayList("admin", "jack")); List dataSetModelConfigs = Lists.newArrayList( - new DataSetModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), - Lists.newArrayList(5L, 6L, 7L), Lists.newArrayList(1L, 2L, 3L, 4L)) + new DataSetModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), Lists.newArrayList(5L, 6L, 7L)) ); DataSetDetail dataSetDetail = new DataSetDetail(); dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs);