From 0858c1336578b3dcc2d8cd81bf22367da5dce76d Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Thu, 4 Jan 2024 12:08:12 +0800 Subject: [PATCH] (improvement)(headless) Update server and core, server calls core one-way (#592) * (improvement)(headless) Update server and core, server calls core one-way * (improvement)(Auth) When obtaining the user information, determine whether the user is a system admin. --------- Co-authored-by: jolunoluo --- .../authentication/service/UserService.java | 4 + .../authentication/rest/UserController.java | 18 +- .../service/UserServiceImpl.java | 28 +- .../match/HanlpDictMatchStrategyTest.java | 18 - .../chat/test/context/ContextTest.java | 37 -- chat/knowledge/pom.xml | 2 +- .../semantic/LocalSemanticInterpreter.java | 14 +- .../headless/api/enums/DatasourceQuery.java} | 6 +- .../headless/api/enums/EngineType.java} | 6 +- .../headless/api/request/QueryS2SQLReq.java | 5 + .../headless/api/request/QueryStructReq.java | 4 + headless/core/pom.xml | 32 -- .../core}/adaptor/db/ClickHouseAdaptor.java | 2 +- .../headless/core}/adaptor/db/DbAdaptor.java | 2 +- .../core/adaptor/db/DbAdaptorFactory.java | 24 ++ .../headless/core}/adaptor/db/H2Adaptor.java | 2 +- .../core}/adaptor/db/MysqlAdaptor.java | 2 +- .../headless/core/executor/JdbcExecutor.java | 9 +- .../headless/core/executor/QueryExecutor.java | 5 +- .../core}/manager/DimensionYamlManager.java | 16 +- .../core/manager/MetricYamlManager.java | 54 +++ .../core}/manager/ModelYamlManager.java | 22 +- .../headless/core/optimizer/DetailQuery.java | 8 +- .../core/optimizer/QueryOptimizer.java | 2 +- .../core/parser/HeadlessConverter.java | 5 +- .../headless/core/parser/QueryParser.java | 44 +-- .../headless/core/parser/SqlParser.java | 5 +- .../core/parser/calcite/CalciteSqlParser.java | 14 +- .../parser/calcite/planner/AggPlanner.java | 24 +- .../planner/MaterializationPlanner.java | 338 ------------------ .../core/parser/calcite/planner/Planner.java | 2 +- .../core/parser/calcite/s2sql/DataSource.java | 5 +- .../parser/calcite/s2sql/HeadlessModel.java | 18 +- .../convert/DefaultDimValueConverter.java | 59 --- .../parser/convert/MetricCheckConverter.java | 86 ----- .../convert/ParserDefaultConverter.java | 80 ----- .../parser/convert/ZipperModelConverter.java | 75 ---- .../CalculateAggConverter.java | 89 ++--- .../converter/ParserDefaultConverter.java | 82 +++++ .../headless/core}/pojo/ConnectInfo.java | 2 +- .../headless/core}/pojo/Database.java | 2 +- .../headless/core}/pojo/JdbcDataSource.java | 4 +- .../pojo/ParserSvrResponse.java | 2 +- .../pojo/QueryStatement.java | 7 +- .../core}/pojo/yaml/DataModelYamlTpl.java | 4 +- .../pojo/yaml/DimensionTimeTypeParamsTpl.java | 2 +- .../core}/pojo/yaml/DimensionYamlTpl.java | 2 +- .../core}/pojo/yaml/IdentifyYamlTpl.java | 2 +- .../core}/pojo/yaml/MeasureYamlTpl.java | 2 +- .../pojo/yaml/MetricTypeParamsYamlTpl.java | 2 +- .../core}/pojo/yaml/MetricYamlTpl.java | 2 +- .../headless/core/utils/ComponentFactory.java | 42 +-- .../core}/utils/JdbcDataSourceUtils.java | 37 +- .../headless/core/utils/SqlGenerateUtils.java | 154 +++++++- .../headless/core}/utils/SqlUtils.java | 4 +- .../core}/utils/SysTimeDimensionBuilder.java | 10 +- .../calcite/MaterializationQueryTest.java | 61 ---- headless/server/pom.xml | 28 +- .../server/adaptor/db/DbAdaptorFactory.java | 24 -- .../server}/annotation/ApiHeaderCheck.java | 2 +- .../annotation/S2SQLDataPermission.java | 2 +- .../annotation/StructDataPermission.java | 2 +- .../server}/aspect/ApiHeaderCheckAspect.java | 4 +- .../server}/aspect/AuthCheckBaseAspect.java | 2 +- .../server}/aspect/DimValueAspect.java | 10 +- .../server}/aspect/S2SQLDataAspect.java | 6 +- .../server}/aspect/StructDataAspect.java | 6 +- .../manager}/HeadlessSchemaManager.java | 28 +- .../server/manager/MetricYamlManager.java | 27 -- .../persistence/mapper/StatMapper.java | 2 +- .../repository/StatRepository.java | 2 +- .../repository/impl}/StatRepositoryImpl.java | 7 +- .../headless/server}/pojo/DataDownload.java | 2 +- .../server}/rest/QueryController.java | 16 +- .../server}/rest/SchemaController.java | 4 +- .../headless/server/service/Catalog.java | 8 +- .../server}/service/DownloadService.java | 2 +- .../server}/service/HeadlessQueryEngine.java | 6 +- .../headless/server/service/ModelService.java | 6 +- .../server}/service/QueryService.java | 16 +- .../server}/service/SchemaService.java | 2 +- .../server/service/impl/CatalogImpl.java | 12 +- .../service/impl/DatabaseServiceImpl.java | 10 +- .../service/impl/DownloadServiceImpl.java | 8 +- .../service/impl/HeadlessQueryEngineImpl.java | 33 +- .../server/service/impl/ModelServiceImpl.java | 12 +- .../service/impl/QueryServiceImpl.java | 48 +-- .../service/impl/SchemaServiceImpl.java | 6 +- .../server/utils/DatabaseConverter.java | 4 +- .../server/utils/DimensionConverter.java | 9 - .../server/utils/MetricConverter.java | 27 -- .../server/utils}/QueryReqConverter.java | 75 ++-- .../server}/utils/QueryStructUtils.java | 212 +---------- .../headless/server}/utils/QueryUtils.java | 5 +- .../headless/server}/utils/StatUtils.java | 14 +- .../src/main/resources/mapper/StatMapper.xml | 2 +- .../calcite/HeadlessParserServiceTest.java | 28 +- .../service/DownloadServiceImplTest.java | 8 +- .../service}/ModelServiceImplTest.java | 8 +- .../server}/utils/DataTransformUtilsTest.java | 3 +- 100 files changed, 811 insertions(+), 1516 deletions(-) delete mode 100644 chat/core/src/test/java/com/tencent/supersonic/chat/mapper/match/HanlpDictMatchStrategyTest.java delete mode 100644 chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java rename headless/{server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatasourceQueryEnum.java => api/src/main/java/com/tencent/supersonic/headless/api/enums/DatasourceQuery.java} (59%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server/pojo/EngineTypeEnum.java => api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java} (74%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/adaptor/db/ClickHouseAdaptor.java (97%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/adaptor/db/DbAdaptor.java (85%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/adaptor/db/H2Adaptor.java (97%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/adaptor/db/MysqlAdaptor.java (97%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/manager/DimensionYamlManager.java (53%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/manager/ModelYamlManager.java (84%) delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/MaterializationPlanner.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/DefaultDimValueConverter.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/MetricCheckConverter.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ParserDefaultConverter.java delete mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ZipperModelConverter.java rename headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/{convert => converter}/CalculateAggConverter.java (85%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/ConnectInfo.java (76%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/Database.java (91%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/JdbcDataSource.java (98%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/{persistence => }/pojo/ParserSvrResponse.java (88%) rename headless/core/src/main/java/com/tencent/supersonic/headless/core/{persistence => }/pojo/QueryStatement.java (86%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/DataModelYamlTpl.java (85%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/DimensionTimeTypeParamsTpl.java (71%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/DimensionYamlTpl.java (84%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/IdentifyYamlTpl.java (82%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/MeasureYamlTpl.java (85%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/MetricTypeParamsYamlTpl.java (73%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/pojo/yaml/MetricYamlTpl.java (79%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/utils/JdbcDataSourceUtils.java (97%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/utils/SqlUtils.java (98%) rename headless/{server/src/main/java/com/tencent/supersonic/headless/server => core/src/main/java/com/tencent/supersonic/headless/core}/utils/SysTimeDimensionBuilder.java (96%) delete mode 100644 headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/MaterializationQueryTest.java delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptorFactory.java rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/annotation/ApiHeaderCheck.java (83%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/annotation/S2SQLDataPermission.java (84%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/annotation/StructDataPermission.java (83%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/aspect/ApiHeaderCheckAspect.java (94%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/aspect/AuthCheckBaseAspect.java (99%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/aspect/DimValueAspect.java (96%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/aspect/S2SQLDataAspect.java (97%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/aspect/StructDataAspect.java (97%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite => server/src/main/java/com/tencent/supersonic/headless/server/manager}/HeadlessSchemaManager.java (92%) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/persistence/mapper/StatMapper.java (83%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/persistence/repository/StatRepository.java (86%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository => server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl}/StatRepositoryImpl.java (93%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core/persistence => server/src/main/java/com/tencent/supersonic/headless/server}/pojo/DataDownload.java (73%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/rest/QueryController.java (95%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/rest/SchemaController.java (96%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/DownloadService.java (90%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/HeadlessQueryEngine.java (85%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/QueryService.java (90%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/SchemaService.java (95%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/impl/DownloadServiceImpl.java (98%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/impl/HeadlessQueryEngineImpl.java (71%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/impl/QueryServiceImpl.java (94%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/service/impl/SchemaServiceImpl.java (96%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core/parser/convert => server/src/main/java/com/tencent/supersonic/headless/server/utils}/QueryReqConverter.java (79%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/utils/QueryStructUtils.java (59%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/utils/QueryUtils.java (98%) rename headless/{core/src/main/java/com/tencent/supersonic/headless/core => server/src/main/java/com/tencent/supersonic/headless/server}/utils/StatUtils.java (98%) rename headless/{core => server}/src/main/resources/mapper/StatMapper.xml (97%) rename headless/{core/src/test/java/com/tencent/supersonic/headless/query/domain => server/src/test/java/com/tencent/supersonic/headless/server}/calcite/HeadlessParserServiceTest.java (92%) rename headless/{core/src/test/java/com/tencent/supersonic/headless/query => server/src/test/java/com/tencent/supersonic/headless/server}/service/DownloadServiceImplTest.java (95%) rename headless/server/src/test/java/com/tencent/supersonic/headless/{model/application => server/service}/ModelServiceImplTest.java (96%) rename headless/{core/src/test/java/com/tencent/supersonic/headless/query => server/src/test/java/com/tencent/supersonic/headless/server}/utils/DataTransformUtilsTest.java (96%) diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java index dcfb3b8ef..09254ccb1 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java @@ -4,11 +4,15 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Set; public interface UserService { + User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse); + List getUserNames(); List getUserList(); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java index 1bbc9e7e1..4a86e92e0 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java @@ -5,18 +5,18 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.service.UserService; -import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import java.util.List; -import java.util.Set; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Set; @RestController @RequestMapping("/api/auth/user") @@ -31,7 +31,7 @@ public class UserController { @GetMapping("/getCurrentUser") public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { - return UserHolder.findUser(httpServletRequest, httpServletResponse); + return userService.getCurrentUser(httpServletRequest, httpServletResponse); } @GetMapping("/getUserNames") diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java index a2d930e03..6cd5c05c5 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java @@ -4,15 +4,39 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.service.UserService; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.authentication.utils.ComponentFactory; +import com.tencent.supersonic.common.pojo.SysParameter; +import com.tencent.supersonic.common.service.SysParameterService; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.util.List; import java.util.Set; -import org.springframework.stereotype.Service; - @Service public class UserServiceImpl implements UserService { + private SysParameterService sysParameterService; + + public UserServiceImpl(SysParameterService sysParameterService) { + this.sysParameterService = sysParameterService; + } + + @Override + public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { + User user = UserHolder.findUser(httpServletRequest, httpServletResponse); + if (user != null) { + SysParameter sysParameter = sysParameterService.getSysParameter(); + if (!CollectionUtils.isEmpty(sysParameter.getAdmins()) + && sysParameter.getAdmins().contains(user.getName())) { + user.setIsAdmin(1); + } + } + return user; + } + @Override public List getUserNames() { return ComponentFactory.getUserAdaptor().getUserNames(); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/match/HanlpDictMatchStrategyTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/match/HanlpDictMatchStrategyTest.java deleted file mode 100644 index 6444822c9..000000000 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/match/HanlpDictMatchStrategyTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.tencent.supersonic.chat.mapper.match; - -import com.tencent.supersonic.chat.test.context.ContextTest; -import org.junit.jupiter.api.Test; - -/** - * MatchStrategyImplTest - */ -class HanlpDictMatchStrategyTest extends ContextTest { - - @Test - void match() { - } - - @Test - void testMatch() { - } -} \ No newline at end of file diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java deleted file mode 100644 index 6d49d1a9b..000000000 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/ContextTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.tencent.supersonic.chat.test.context; - -import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper; -import com.tencent.supersonic.chat.persistence.repository.impl.ChatContextRepositoryImpl; -import com.tencent.supersonic.chat.test.ChatBizLauncher; -import com.tencent.supersonic.chat.utils.ComponentFactory; -import com.tencent.supersonic.headless.core.service.QueryService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.knowledge.semantic.RemoteSemanticInterpreter; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.web.client.RestTemplate; - -@MockBean(ChatContextRepositoryImpl.class) -@MockBean(QueryService.class) -@MockBean(DimensionService.class) -@MockBean(MetricService.class) -@MockBean(ModelService.class) -@MockBean(ChatContextMapper.class) -@MockBean(RestTemplate.class) -@MockBean(RemoteSemanticInterpreter.class) -@MockBean(ComponentFactory.class) -//@MybatisTest -//@AutoConfigureMybatis -//@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ChatBizLauncher.class) -public class ContextTest { - - protected final Logger logger = LoggerFactory.getLogger(ContextTest.class); -} diff --git a/chat/knowledge/pom.xml b/chat/knowledge/pom.xml index 197a134e7..14cec57ad 100644 --- a/chat/knowledge/pom.xml +++ b/chat/knowledge/pom.xml @@ -100,7 +100,7 @@ com.tencent.supersonic - headless-core + headless-server ${project.version} compile diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java index 72a70856f..4d8fa671c 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java @@ -5,9 +5,14 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; +import com.tencent.supersonic.headless.api.request.QueryDimValueReq; +import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; +import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.ExplainResp; @@ -15,15 +20,10 @@ import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.api.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.request.QueryDimValueReq; -import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.service.QueryService; -import com.tencent.supersonic.headless.core.service.SchemaService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatasourceQueryEnum.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/DatasourceQuery.java similarity index 59% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatasourceQueryEnum.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/DatasourceQuery.java index 7a3aae813..1f2aee525 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatasourceQueryEnum.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/DatasourceQuery.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.headless.server.pojo; +package com.tencent.supersonic.headless.api.enums; -public enum DatasourceQueryEnum { +public enum DatasourceQuery { SQL_QUERY("sql_query"), TABLE_QUERY("table_query"); @@ -9,7 +9,7 @@ public enum DatasourceQueryEnum { private String name; - DatasourceQueryEnum(String name) { + DatasourceQuery(String name) { this.name = name; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/EngineTypeEnum.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java similarity index 74% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/EngineTypeEnum.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java index aafc40eee..563b5ab98 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/EngineTypeEnum.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.headless.server.pojo; +package com.tencent.supersonic.headless.api.enums; -public enum EngineTypeEnum { +public enum EngineType { TDW(0, "tdw"), MYSQL(1, "mysql"), @@ -15,7 +15,7 @@ public enum EngineTypeEnum { private String name; - EngineTypeEnum(Integer code, String name) { + EngineType(Integer code, String name) { this.code = code; this.name = name; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java index cd8bce535..51d2c9c45 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java @@ -7,6 +7,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Data @ToString @@ -27,4 +28,8 @@ public class QueryS2SQLReq { return Lists.newArrayList(modelIds); } + public String getModelIdStr() { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java index 8979b458f..039a20334 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java @@ -82,6 +82,10 @@ public class QueryStructReq { return Lists.newArrayList(modelIds); } + public String getModelIdStr() { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + } + public Set getModelIdSet() { return modelIds; } diff --git a/headless/core/pom.xml b/headless/core/pom.xml index 1e4286ea3..4d49a1ee8 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -30,12 +30,6 @@ spring-context - - - com.tencent.supersonic - common - ${project.version} - com.tencent.supersonic headless-api @@ -60,12 +54,6 @@ ${jetty.util.version} compile - - com.tencent.supersonic - headless-server - ${project.version} - compile - com.jayway.jsonpath json-path @@ -87,26 +75,6 @@ - - com.alibaba - easyexcel - ${easyexcel.version} - - - org.apache.poi - poi - ${poi.version} - - - org.apache.poi - poi-ooxml - ${poi.version} - - - org.apache.poi - poi-ooxml-schemas - ${poi.version} - diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/ClickHouseAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java similarity index 97% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/ClickHouseAdaptor.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java index fd99426c5..fd9524313 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/ClickHouseAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.adaptor.db; +package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java similarity index 85% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptor.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index 4e8f7e17c..a76b39a86 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.adaptor.db; +package com.tencent.supersonic.headless.core.adaptor.db; public abstract class DbAdaptor { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java new file mode 100644 index 000000000..ee93047d7 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java @@ -0,0 +1,24 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.tencent.supersonic.headless.api.enums.EngineType; + +import java.util.HashMap; +import java.util.Map; + + +public class DbAdaptorFactory { + + private static Map dbAdaptorMap; + + static { + dbAdaptorMap = new HashMap<>(); + dbAdaptorMap.put(EngineType.CLICKHOUSE.getName(), new ClickHouseAdaptor()); + dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor()); + dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor()); + } + + public static DbAdaptor getEngineAdaptor(String engineType) { + return dbAdaptorMap.get(engineType); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/H2Adaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java similarity index 97% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/H2Adaptor.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java index d2c47a1d9..3fca71a74 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/H2Adaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.adaptor.db; +package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/MysqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java similarity index 97% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/MysqlAdaptor.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java index d8a890cef..6d08905fa 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/MysqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.adaptor.db; +package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index 6f3994f44..6e3e8222d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -2,9 +2,8 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.service.Catalog; -import com.tencent.supersonic.headless.server.utils.SqlUtils; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.SqlUtils; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; @@ -25,13 +24,13 @@ public class JdbcExecutor implements QueryExecutor { } @Override - public QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement) { + public QueryResultWithSchemaResp execute(QueryStatement queryStatement) { if (Strings.isEmpty(queryStatement.getSourceId())) { log.warn("data base id is empty"); return null; } log.info("query SQL: {}", queryStatement.getSql()); - DatabaseResp databaseResp = catalog.getDatabase(Long.parseLong(queryStatement.getSourceId())); + DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); log.info("database info:{}", databaseResp); QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java index de6e493ce..201df3b37 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; public interface QueryExecutor { boolean accept(QueryStatement queryStatement); - QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement); + QueryResultWithSchemaResp execute(QueryStatement queryStatement); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/DimensionYamlManager.java similarity index 53% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/DimensionYamlManager.java index eae44159f..43f747871 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/DimensionYamlManager.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.headless.server.manager; +package com.tencent.supersonic.headless.core.manager; import com.tencent.supersonic.headless.api.enums.IdentifyType; import com.tencent.supersonic.headless.api.response.DimensionResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.utils.DimensionConverter; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -24,7 +24,15 @@ public class DimensionYamlManager { } return dimensions.stream() .filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name())) - .map(DimensionConverter::convert2DimensionYamlTpl).collect(Collectors.toList()); + .map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList()); + } + + public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) { + DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl(); + BeanUtils.copyProperties(dimension, dimensionYamlTpl); + dimensionYamlTpl.setName(dimension.getBizName()); + dimensionYamlTpl.setOwners(dimension.getCreatedBy()); + return dimensionYamlTpl; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java new file mode 100644 index 000000000..3cea1df9d --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java @@ -0,0 +1,54 @@ +package com.tencent.supersonic.headless.core.manager; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.Measure; +import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; +import com.tencent.supersonic.headless.api.response.MetricResp; +import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + + +@Slf4j +@Service +public class MetricYamlManager { + + public static List convert2YamlObj(List metrics) { + + List metricYamlTpls = new ArrayList<>(); + for (MetricResp metric : metrics) { + MetricYamlTpl metricYamlTpl = convert2MetricYamlTpl(metric); + metricYamlTpls.add(metricYamlTpl); + } + return metricYamlTpls; + } + + public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) { + MetricYamlTpl metricYamlTpl = new MetricYamlTpl(); + BeanUtils.copyProperties(metric, metricYamlTpl); + metricYamlTpl.setName(metric.getBizName()); + metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); + MetricTypeParams exprMetricTypeParams = metric.getTypeParams(); + MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); + metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr()); + List measures = exprMetricTypeParams.getMeasures(); + metricTypeParamsYamlTpl.setMeasures( + measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); + metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl); + return metricYamlTpl; + } + + public static MeasureYamlTpl convert(Measure measure) { + MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl(); + measureYamlTpl.setName(measure.getBizName()); + return measureYamlTpl; + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java similarity index 84% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java index 212ee045a..b26d61c70 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java @@ -1,5 +1,6 @@ -package com.tencent.supersonic.headless.server.manager; +package com.tencent.supersonic.headless.core.manager; +import com.tencent.supersonic.headless.api.enums.DatasourceQuery; import com.tencent.supersonic.headless.api.enums.ModelSourceType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Identify; @@ -7,15 +8,14 @@ import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.server.pojo.DatasourceQueryEnum; -import com.tencent.supersonic.headless.server.utils.SysTimeDimensionBuilder; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -45,7 +45,7 @@ public class ModelYamlManager { dataModelYamlTpl.setName(modelResp.getBizName()); dataModelYamlTpl.setSourceId(modelResp.getDatabaseId()); dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType())); - if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQueryEnum.SQL_QUERY.getName())) { + if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) { dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery()); } else { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java index 84ace0418..36663137a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java @@ -2,14 +2,14 @@ package com.tencent.supersonic.headless.core.optimizer; import com.google.common.base.Strings; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; - -import java.util.Objects; -import java.util.stream.Collectors; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.Objects; +import java.util.stream.Collectors; + @Slf4j @Component("DetailQuery") public class DetailQuery implements QueryOptimizer { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java index 11e51c659..ea1ba0db2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.core.optimizer; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; public interface QueryOptimizer { void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java index f06b22fa8..053144157 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java @@ -1,12 +1,11 @@ package com.tencent.supersonic.headless.core.parser; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; public interface HeadlessConverter { boolean accept(QueryStatement queryStatement); - void converter(Catalog catalog, QueryStatement queryStatement) throws Exception; + void convert(QueryStatement queryStatement) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java index 865e7c265..be09491ed 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java @@ -6,9 +6,8 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import com.tencent.supersonic.headless.server.service.Catalog; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Primary; @@ -25,13 +24,6 @@ import java.util.stream.Collectors; @Primary public class QueryParser { - - private final Catalog catalog; - - public QueryParser(Catalog catalog) { - this.catalog = catalog; - } - public QueryStatement logicSql(QueryStatement queryStatement) throws Exception { QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); if (Objects.isNull(queryStatement.getParseSqlReq())) { @@ -44,7 +36,7 @@ public class QueryParser { for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) { if (headlessConverter.accept(queryStatement)) { log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName()); - headlessConverter.converter(catalog, queryStatement); + headlessConverter.convert(queryStatement); } } log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(), @@ -58,24 +50,26 @@ public class QueryParser { } - public QueryStatement parser(ParseSqlReq sqlCommend, QueryStatement queryStatement) { - log.info("parser MetricReq [{}] ", sqlCommend); + public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) { + log.info("parser MetricReq [{}] ", parseSqlReq); try { - if (!CollectionUtils.isEmpty(sqlCommend.getTables())) { + if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) { List tables = new ArrayList<>(); String sourceId = ""; - for (MetricTable metricTable : sqlCommend.getTables()) { + for (MetricTable metricTable : parseSqlReq.getTables()) { MetricQueryReq metricReq = new MetricQueryReq(); metricReq.setMetrics(metricTable.getMetrics()); metricReq.setDimensions(metricTable.getDimensions()); metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); - metricReq.setRootPath(sqlCommend.getRootPath()); + metricReq.setRootPath(parseSqlReq.getRootPath()); QueryStatement tableSql = new QueryStatement(); tableSql.setIsS2SQL(false); tableSql.setMetricReq(metricReq); tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); + tableSql.setModelIds(queryStatement.getModelIds()); + tableSql.setHeadlessModel(queryStatement.getHeadlessModel()); tableSql = parser(tableSql, metricTable.getAggOption()); if (!tableSql.isOk()) { queryStatement.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(), @@ -88,20 +82,20 @@ public class QueryParser { if (!tables.isEmpty()) { String sql = ""; - if (sqlCommend.isSupportWith()) { + if (parseSqlReq.isSupportWith()) { sql = "with " + String.join(",", tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect( - Collectors.toList())) + "\n" + sqlCommend.getSql(); + Collectors.toList())) + "\n" + parseSqlReq.getSql(); } else { - sql = sqlCommend.getSql(); + sql = parseSqlReq.getSql(); for (String[] tb : tables) { sql = StringUtils.replace(sql, tb[0], - "(" + tb[1] + ") " + (sqlCommend.isWithAlias() ? "" : tb[0]), -1); + "(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1); } } queryStatement.setSql(sql); queryStatement.setSourceId(sourceId); - queryStatement.setParseSqlReq(sqlCommend); + queryStatement.setParseSqlReq(parseSqlReq); return queryStatement; } } @@ -117,18 +111,18 @@ public class QueryParser { } public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) { - MetricQueryReq metricCommand = queryStatement.getMetricReq(); - log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg); - if (metricCommand.getRootPath().isEmpty()) { + MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); + log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg); + if (metricQueryReq.getRootPath().isEmpty()) { queryStatement.setErrMsg("rootPath empty"); return queryStatement; } try { - queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg, catalog); + queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg); return queryStatement; } catch (Exception e) { queryStatement.setErrMsg(e.getMessage()); - log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e); + log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e); } return queryStatement; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/SqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/SqlParser.java index 33b5e3042..fd5dbc93d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/SqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/SqlParser.java @@ -1,10 +1,9 @@ package com.tencent.supersonic.headless.core.parser; import com.tencent.supersonic.headless.api.enums.AggOption; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; public interface SqlParser { - QueryStatement explain(QueryStatement queryStatement, AggOption aggOption, Catalog catalog) throws Exception; + QueryStatement explain(QueryStatement queryStatement, AggOption aggOption) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java index 9ff84c7f0..4211911d2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java @@ -7,24 +7,16 @@ import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import org.springframework.stereotype.Component; @Component("CalciteSqlParser") public class CalciteSqlParser implements SqlParser { - private final HeadlessSchemaManager headlessSchemaManager; - - public CalciteSqlParser( - HeadlessSchemaManager headlessSchemaManager) { - this.headlessSchemaManager = headlessSchemaManager; - } - @Override - public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg, Catalog catalog) throws Exception { + public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception { MetricQueryReq metricReq = queryStatement.getMetricReq(); - HeadlessModel headlessModel = headlessSchemaManager.get(metricReq.getRootPath()); + HeadlessModel headlessModel = queryStatement.getHeadlessModel(); if (headlessModel == null) { queryStatement.setErrMsg("semanticSchema not found"); return queryStatement; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java index feb20c309..006e6d266 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java @@ -4,27 +4,20 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner; import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.core.parser.calcite.Configuration; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer; import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender; import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Objects; -import java.util.Stack; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.hep.HepPlanner; @@ -36,6 +29,13 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql2rel.SqlToRelConverter; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Objects; +import java.util.Stack; + @Slf4j public class AggPlanner implements Planner { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/MaterializationPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/MaterializationPlanner.java deleted file mode 100644 index 753013c5d..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/MaterializationPlanner.java +++ /dev/null @@ -1,338 +0,0 @@ -package com.tencent.supersonic.headless.core.parser.calcite.planner; - -import com.tencent.supersonic.common.util.calcite.SqlParseUtils; -import com.tencent.supersonic.headless.api.enums.AggOption; -import com.tencent.supersonic.headless.api.request.MetricQueryReq; -import com.tencent.supersonic.headless.core.parser.calcite.Configuration; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MaterializationElement; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.TimeRange; -import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.adapter.enumerable.EnumerableRules; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptMaterialization; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgramBuilder; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelDistributionTraitDef; -import org.apache.calcite.rel.RelHomogeneousShuttle; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttle; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.rules.materialize.MaterializedViewRules; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.tools.RelBuilder; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.springframework.util.CollectionUtils; - -@Slf4j -public class MaterializationPlanner implements Planner { - - protected HeadlessSchema schema; - protected CalciteSchema viewSchema; - protected HepProgramBuilder hepProgramBuilder; - protected RelOptPlanner relOptPlanner; - protected RelBuilder relBuilder; - protected CalciteCatalogReader calciteCatalogReader; - - protected Comparator materializationSort = new Comparator>>() { - @Override - public int compare(Entry> o1, Entry> o2) { - if (o1.getValue().size() == o2.getValue().size()) { - Optional o1Lever = schema.getMaterializationList().stream() - .filter(m -> m.getMaterializationId().equals(o1.getKey())).findFirst(); - Optional o2Lever = schema.getMaterializationList().stream() - .filter(m -> m.getMaterializationId().equals(o2.getKey())).findFirst(); - if (o1Lever.isPresent() && o2Lever.isPresent()) { - return o2Lever.get().getLevel() - o1Lever.get().getLevel(); - } - return 0; - } - return o2.getValue().size() - o1.getValue().size(); - } - }; - - public MaterializationPlanner(HeadlessSchema schema) { - this.schema = schema; - init(); - } - - @Override - public void explain(QueryStatement queryStatement, AggOption isAgg) throws Exception { - // findMatchMaterialization - // checkValid field + time - if (CollectionUtils.isEmpty(queryStatement.getTimeRanges())) { - //has no matchMaterialization time info - return; - } - Set fields = new HashSet<>(); - MetricQueryReq metricCommand = queryStatement.getMetricReq(); - if (!Objects.isNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { - fields.addAll(SqlParseUtils.getFilterField(metricCommand.getWhere())); - } - if (!CollectionUtils.isEmpty(metricCommand.getMetrics())) { - fields.addAll(metricCommand.getMetrics()); - } - if (!CollectionUtils.isEmpty(metricCommand.getDimensions())) { - fields.addAll(metricCommand.getDimensions()); - } - Map> matchMaterialization = new HashMap<>(); - Map materializationDataBase = schema.getMaterializationList().stream() - .collect(Collectors.toMap(Materialization::getMaterializationId, Materialization::getDataBase)); - for (String elem : fields) { - boolean checkOk = false; - for (Materialization materialization : schema.getMaterializationList()) { - if (check(metricCommand, materialization, elem, queryStatement.getTimeRanges())) { - if (!matchMaterialization.containsKey(materialization.getMaterializationId())) { - matchMaterialization.put(materialization.getMaterializationId(), new HashSet<>()); - } - matchMaterialization.get(materialization.getMaterializationId()).add(elem); - checkOk = true; - } - } - if (!checkOk) { - log.info("check fail [{}]", elem); - } - } - if (!CollectionUtils.isEmpty(matchMaterialization)) { - List>> sortedMaterialization = new ArrayList<>(matchMaterialization.entrySet()); - sortedMaterialization.stream().collect(Collectors.toList()).sort(materializationSort); - for (Entry> m : sortedMaterialization) { - Optional materialization = schema.getMaterializationList().stream() - .filter(mz -> mz.getMaterializationId().equals(m.getKey())).findFirst(); - if (!materialization.isPresent()) { - continue; - } - Set viewField = new HashSet<>(m.getValue()); - viewField.add(materialization.get().getEntities()); - viewField.add(materialization.get().getDateInfo()); - if (materialization.get().getTimePartType().equals(TimePartType.ZIPPER)) { - viewField.add(Constants.MATERIALIZATION_ZIPPER_START + materialization.get().getDateInfo()); - viewField.add(Constants.MATERIALIZATION_ZIPPER_END + materialization.get().getDateInfo()); - } - if (viewField.containsAll(fields)) { - addDataSource(materialization.get()); - break; - } - List>> linkMaterialization = new ArrayList<>(); - for (Entry> mm : sortedMaterialization) { - if (mm.getKey().equals(m.getKey())) { - continue; - } - if (materializationDataBase.get(mm.getKey()).equals(materializationDataBase.get(m.getKey()))) { - linkMaterialization.add(mm); - } - } - if (!CollectionUtils.isEmpty(linkMaterialization)) { - linkMaterialization.sort(materializationSort); - for (Entry> mm : linkMaterialization) { - Set linkField = new HashSet<>(mm.getValue()); - linkField.addAll(viewField); - if (linkField.containsAll(fields)) { - Optional linkMaterial = schema.getMaterializationList().stream() - .filter(mz -> mz.getMaterializationId().equals(mm.getKey())).findFirst(); - if (linkMaterial.isPresent()) { - addDataSource(materialization.get()); - addDataSource(linkMaterial.get()); - break; - } - } - } - } - } - } - } - - private void addDataSource(Materialization materialization) { - Identify identify = new Identify(); - identify.setName(materialization.getEntities()); - List metrics = materialization.getMetrics().stream() - .map(m -> Measure.builder().name(m.getName()).expr(m.getName()).build()).collect( - Collectors.toList()); - List dimensions = materialization.getDimensions().stream() - .map(d -> Dimension.builder().name(d.getName()).expr(d.getName()).build()).collect( - Collectors.toList()); - if (materialization.getTimePartType().equals(TimePartType.ZIPPER)) { - dimensions.add( - Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo()) - .type(Constants.DIMENSION_TYPE_TIME) - .expr(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo()).build()); - dimensions.add( - Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo()) - .type(Constants.DIMENSION_TYPE_TIME) - .expr(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo()).build()); - } else { - dimensions.add(Dimension.builder().name(materialization.getDateInfo()).expr(materialization.getDateInfo()) - .type(Constants.DIMENSION_TYPE_TIME) - .build()); - } - - DataSource dataSource = DataSource.builder().sourceId(materialization.getDataBase()) - .tableQuery(materialization.getDestinationTable()) - .timePartType(materialization.getTimePartType()) - .name("v_" + String.valueOf(materialization.getMaterializationId())) - .identifiers(Arrays.asList(identify)) - .measures(metrics) - .dimensions(dimensions) - .build(); - schema.getDatasource().put(dataSource.getName(), dataSource); - } - - @Override - public String getSql() { - return null; - } - - @Override - public String getSourceId() { - return null; - } - - @Override - public HeadlessSchema findBest() { - return schema; - } - - private void init() { - viewSchema = SchemaBuilder.getMaterializationSchema(); - hepProgramBuilder = new HepProgramBuilder(); - hepProgramBuilder.addRuleInstance(MaterializedViewRules.PROJECT_FILTER); - relOptPlanner = new HepPlanner(hepProgramBuilder.build()); - calciteCatalogReader = new CalciteCatalogReader( - CalciteSchema.from(viewSchema.plus()), - CalciteSchema.from(viewSchema.plus()).path(null), - Configuration.typeFactory, - new CalciteConnectionConfigImpl(new Properties())); - - relOptPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - relOptPlanner.addRelTraitDef(RelDistributionTraitDef.INSTANCE); - EnumerableRules.rules().forEach(relOptPlanner::addRule); - - RexBuilder rexBuilder = new RexBuilder(Configuration.typeFactory); - RelOptCluster relOptCluster = RelOptCluster.create(relOptPlanner, rexBuilder); - relBuilder = RelFactories.LOGICAL_BUILDER.create(relOptCluster, calciteCatalogReader); - } - - private RexNode getRexNode(List> timeRanges, String viewField) { - RexNode rexNode = null; - for (ImmutablePair timeRange : timeRanges) { - if (rexNode == null) { - rexNode = getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(), - viewField); - continue; - } - rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, - getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(), - viewField)); - } - return rexNode; - } - - private RexNode getRexNode(Materialization materialization, String elem, String viewField) { - Optional dim = materialization.getDimensions() - .stream().filter(d -> d.getName().equalsIgnoreCase(elem)).findFirst(); - if (!dim.isPresent()) { - dim = materialization.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(elem)).findFirst(); - } - RexNode rexNode = null; - if (dim.isPresent()) { - for (TimeRange timeRange : dim.get().getTimeRangeList()) { - if (rexNode == null) { - rexNode = getRexNodeByTimeRange(timeRange, viewField); - continue; - } - rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, getRexNodeByTimeRange(timeRange, viewField)); - } - } - return rexNode; - } - - private RexNode getRexNodeByTimeRange(TimeRange timeRange, String field) { - return relBuilder.call(SqlStdOperatorTable.AND, - relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(field), - relBuilder.literal(timeRange.getStart())), - relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(field), - relBuilder.literal(timeRange.getEnd()))); - } - - public boolean check(MetricQueryReq metricCommand, Materialization materialization, String elem, - List> timeRanges) - throws SqlParseException { - boolean isMatch = false; - try { - relBuilder.clear(); - if (!CollectionUtils.isEmpty(relOptPlanner.getMaterializations())) { - relOptPlanner.clear(); - } - String db = SchemaBuilder.MATERIALIZATION_SYS_DB; - RelBuilder viewBuilder = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE)); - RexNode viewFilter = getRexNode(materialization, elem, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE); - if (viewFilter == null) { - return false; - } - RelNode viewRel = viewBuilder.filter(viewFilter).project(relBuilder.fields()).build(); - log.debug("view {}", viewRel.explain()); - List view = Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_VIEW); - RelNode replacement = relBuilder.scan(view).build(); - RelOptMaterialization relOptMaterialization = new RelOptMaterialization(replacement, viewRel, null, view); - relOptPlanner.addMaterialization(relOptMaterialization); - - RelNode checkRel = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE)) - .filter(getRexNode(timeRanges, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE)) - .project(relBuilder.field(SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE)).build(); - - relOptPlanner.setRoot(checkRel); - RelNode optRel = relOptPlanner.findBestExp(); - log.debug("findBestExp {}", optRel.explain()); - isMatch = !extractTableNames(optRel).contains(SchemaBuilder.MATERIALIZATION_SYS_SOURCE); - } catch (Exception e) { - log.error("check error {}", e); - } - return isMatch; - } - - public static Set extractTableNames(RelNode relNode) { - Set tableNames = new HashSet<>(); - RelShuttle shuttle = new RelHomogeneousShuttle() { - public RelNode visit(TableScan scan) { - RelOptTable table = scan.getTable(); - tableNames.addAll(table.getQualifiedName()); - return scan; - } - }; - relNode.accept(shuttle); - return tableNames; - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java index 5bfeee00e..9797bf958 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner; import com.tencent.supersonic.headless.api.enums.AggOption; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; public interface Planner { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java index ae1fb27cd..c68bb9810 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java @@ -1,14 +1,17 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; -import java.util.List; import lombok.Builder; import lombok.Data; +import java.util.List; + @Data @Builder public class DataSource { + private Long id; + private String name; private Long sourceId; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java index f7736864b..31aec10cd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java @@ -1,10 +1,15 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; +import com.tencent.supersonic.common.pojo.ItemDateResp; +import com.tencent.supersonic.headless.api.response.DatabaseResp; +import lombok.Data; + import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.Data; +import java.util.stream.Collectors; @Data public class HeadlessModel { @@ -15,4 +20,15 @@ public class HeadlessModel { private Map> dimensionMap = new HashMap<>(); private List materializationList = new ArrayList<>(); private List joinRelations; + private ItemDateResp dataDate; + private DatabaseResp databaseResp; + + public List getDimensions() { + return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + } + + public Map getModelMap() { + return datasourceMap.values().stream().collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource)); + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/DefaultDimValueConverter.java deleted file mode 100644 index 9d6134264..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/DefaultDimValueConverter.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.tencent.supersonic.headless.core.parser.convert; - -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.DimensionResp; -import com.tencent.supersonic.headless.core.parser.HeadlessConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.Catalog; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -@Slf4j -@Component("DefaultDimValueConverter") -public class DefaultDimValueConverter implements HeadlessConverter { - - @Override - public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { - return false; - } - return true; - } - - @Override - public void converter(Catalog catalog, QueryStatement queryStatement) { - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds()); - List dimensionResps = catalog.getDimensions(metaFilter); - //dimension which has default values - dimensionResps = dimensionResps.stream() - .filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues())) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(dimensionResps)) { - return; - } - log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd); - //add dimension default value to filter - List dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream() - .map(Filter::getBizName).collect(Collectors.toList()); - for (DimensionResp dimensionResp : dimensionResps) { - if (!dimensionFilterBizName.contains(dimensionResp.getBizName())) { - Filter filter = new Filter(); - filter.setBizName(dimensionResp.getBizName()); - filter.setValue(dimensionResp.getDefaultValues()); - filter.setOperator(FilterOperatorEnum.IN); - filter.setName(dimensionResp.getName()); - queryStructCmd.getDimensionFilters().add(filter); - } - } - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/MetricCheckConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/MetricCheckConverter.java deleted file mode 100644 index 36be26857..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/MetricCheckConverter.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.tencent.supersonic.headless.core.parser.convert; - -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.DimensionResp; -import com.tencent.supersonic.headless.api.response.MetricResp; -import com.tencent.supersonic.headless.core.parser.HeadlessConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.Catalog; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - - -@Component("MetricCheckConverter") -@Slf4j -public class MetricCheckConverter implements HeadlessConverter { - - @Override - public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { - return false; - } - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - if (queryStructCmd.getQueryType().isNativeAggQuery()) { - return false; - } - return !CollectionUtils.isEmpty(queryStructCmd.getAggregators()); - } - - @Override - public void converter(Catalog catalog, QueryStatement queryStatement) { - QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); - MetaFilter metaFilter = new MetaFilter(queryStructReq.getModelIds()); - List metricResps = catalog.getMetrics(metaFilter); - List dimensionResps = catalog.getDimensions(metaFilter); - Map dimensionMap = dimensionResps.stream() - .collect(Collectors.toMap(DimensionResp::getId, d -> d)); - List metricBizNames = queryStructReq.getMetrics(); - List dimensionFilterBizNames = queryStructReq.getDimensionFilters().stream() - .map(Filter::getBizName).collect(Collectors.toList()); - List metricToQuery = metricResps.stream().filter(metricResp -> - metricBizNames.contains(metricResp.getBizName())).collect(Collectors.toList()); - List dimensionToFilter = dimensionResps.stream().filter(dimensionResp -> - dimensionFilterBizNames.contains(dimensionResp.getBizName())) - .map(DimensionResp::getId).collect(Collectors.toList()); - for (MetricResp metricResp : metricToQuery) { - Set necessaryDimensionIds = metricResp.getNecessaryDimensionIds(); - if (CollectionUtils.isEmpty(necessaryDimensionIds)) { - continue; - } - DimensionResp dimensionResp = null; - for (Long dimensionId : necessaryDimensionIds) { - dimensionResp = dimensionMap.get(dimensionId); - if (dimensionResp != null) { - break; - } - } - if (dimensionResp == null) { - continue; - } - String message = String.format("该指标必须配合维度[%s]来进行过滤查询", dimensionResp.getName()); - if (CollectionUtils.isEmpty(dimensionToFilter)) { - throw new InvalidArgumentException(message); - } - boolean flag = false; - for (Long dimensionId : dimensionToFilter) { - if (necessaryDimensionIds.contains(dimensionId)) { - flag = true; - break; - } - } - if (!flag) { - throw new InvalidArgumentException(message); - } - } - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ParserDefaultConverter.java deleted file mode 100644 index a8b2c72f6..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ParserDefaultConverter.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.tencent.supersonic.headless.core.parser.convert; - -import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.headless.api.pojo.Param; -import com.tencent.supersonic.headless.api.request.MetricQueryReq; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.parser.HeadlessConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; -import com.tencent.supersonic.headless.server.service.Catalog; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.Objects; -import java.util.stream.Collectors; - -@Component("ParserDefaultConverter") -@Slf4j -public class ParserDefaultConverter implements HeadlessConverter { - - - private final CalculateAggConverter calculateCoverterAgg; - private final QueryStructUtils queryStructUtils; - - public ParserDefaultConverter( - CalculateAggConverter calculateCoverterAgg, - QueryStructUtils queryStructUtils) { - this.calculateCoverterAgg = calculateCoverterAgg; - this.queryStructUtils = queryStructUtils; - } - - @Override - public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { - return false; - } - return !calculateCoverterAgg.accept(queryStatement); - } - - @Override - public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception { - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - MetricQueryReq metricCommand = queryStatement.getMetricReq(); - MetricQueryReq metricReq = generateSqlCommand(catalog, queryStructCmd); - queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd)); - BeanUtils.copyProperties(metricReq, metricCommand); - } - - public MetricQueryReq generateSqlCommand(Catalog catalog, QueryStructReq queryStructCmd) { - MetricQueryReq sqlCommend = new MetricQueryReq(); - sqlCommend.setMetrics(queryStructCmd.getMetrics()); - sqlCommend.setDimensions(queryStructCmd.getGroups()); - String where = queryStructUtils.generateWhere(queryStructCmd); - log.info("in generateSqlCommend, complete where:{}", where); - - sqlCommend.setWhere(where); - sqlCommend.setOrder(queryStructCmd.getOrders().stream() - .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList())); - sqlCommend.setVariables(queryStructCmd.getParams().stream() - .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); - sqlCommend.setLimit(queryStructCmd.getLimit()); - String rootPath = catalog.getModelFullPath(queryStructCmd.getModelIds()); - sqlCommend.setRootPath(rootPath); - - // todo tmp delete - // support detail query - if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) { - for (Long modelId : queryStructCmd.getModelIds()) { - String internalMetricName = queryStructUtils.generateInternalMetricName( - modelId, queryStructCmd.getGroups()); - sqlCommend.getMetrics().add(internalMetricName); - } - } - - return sqlCommend; - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ZipperModelConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ZipperModelConverter.java deleted file mode 100644 index 58d5e2b5f..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ZipperModelConverter.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.tencent.supersonic.headless.core.parser.convert; - -import com.tencent.supersonic.headless.api.request.MetricQueryReq; -import com.tencent.supersonic.headless.api.request.ParseSqlReq; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.enums.ModelSourceType; -import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.core.parser.HeadlessConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; -import com.tencent.supersonic.headless.server.service.Catalog; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.stereotype.Component; - -import java.util.List; -import java.util.Objects; - -/** - * correct the Query parameters when the model source type is zipper - */ -@Component("ZipperModelConverter") -@Slf4j -public class ZipperModelConverter implements HeadlessConverter { - - private final QueryStructUtils queryStructUtils; - private final Catalog catalog; - - public ZipperModelConverter(QueryStructUtils queryStructUtils, - Catalog catalog) { - this.queryStructUtils = queryStructUtils; - this.catalog = catalog; - } - - @Override - public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryStructReq())) { - return false; - } - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - List modelRespList = catalog.getModelList(queryStructCmd.getModelIds()); - if (!CollectionUtils.isEmpty(modelRespList)) { - // all data sources are zipper tables - long zipperCnt = modelRespList.stream().filter(m -> ModelSourceType.isZipper(m.getSourceType())) - .count(); - return modelRespList.size() == zipperCnt; - } - return false; - } - - @Override - public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception { - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); - ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); - MetricQueryReq metricCommand = queryStatement.getMetricReq(); - doSingleZipperSource(queryStructCmd, sqlCommend, metricCommand); - } - - protected void doSingleZipperSource(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, - MetricQueryReq metricCommand) { - // all data sources are zipper tables - // request time field rewrite to start_ end_ - if (!sqlCommend.getSql().isEmpty()) { - String sqlNew = queryStructUtils.generateZipperWhere(queryStructCmd, sqlCommend); - log.info("doSingleZipperSource before[{}] after[{}]", sqlCommend.getSql(), sqlNew); - sqlCommend.setSql(sqlNew); - } else { - String where = queryStructUtils.generateZipperWhere(queryStructCmd); - if (!where.isEmpty() && Objects.nonNull(metricCommand)) { - log.info("doSingleZipperSource before[{}] after[{}]", metricCommand.getWhere(), where); - metricCommand.setWhere(where); - } - } - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java similarity index 85% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/CalculateAggConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index eaa72879c..1fd4c208d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -1,5 +1,4 @@ -package com.tencent.supersonic.headless.core.parser.convert; - +package com.tencent.supersonic.headless.core.parser.converter; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; @@ -7,20 +6,15 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.core.parser.HeadlessConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum; -import com.tencent.supersonic.headless.server.service.Catalog; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -35,24 +29,11 @@ import java.util.stream.Collectors; @Slf4j public class CalculateAggConverter implements HeadlessConverter { - - private final HeadlessQueryEngine parserService; - private final QueryStructUtils queryStructUtils; private final SqlGenerateUtils sqlGenerateUtils; - private final Catalog catalog; - - @Value("${metricParser.agg.default:sum}") - private String metricAggDefault; - public CalculateAggConverter( - HeadlessQueryEngine parserService, - @Lazy QueryStructUtils queryStructUtils, - SqlGenerateUtils sqlGenerateUtils, Catalog catalog) { - this.parserService = parserService; - this.queryStructUtils = queryStructUtils; + public CalculateAggConverter(SqlGenerateUtils sqlGenerateUtils) { this.sqlGenerateUtils = sqlGenerateUtils; - this.catalog = catalog; } public interface EngineSql { @@ -60,34 +41,36 @@ public class CalculateAggConverter implements HeadlessConverter { String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql); } - public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum, String version) + public ParseSqlReq generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version) throws Exception { + QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); // 同环比 - if (isRatioAccept(queryStructCmd)) { - return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version); + if (isRatioAccept(queryStructReq)) { + return generateRatioSqlCommand(queryStatement, engineTypeEnum, version); } ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds())); + sqlCommand.setRootPath(queryStructReq.getModelIdStr()); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); - metricTable.setMetrics(queryStructCmd.getMetrics()); - metricTable.setDimensions(queryStructCmd.getGroups()); - String where = queryStructUtils.generateWhere(queryStructCmd); + metricTable.setMetrics(queryStructReq.getMetrics()); + metricTable.setDimensions(queryStructReq.getGroups()); + String where = sqlGenerateUtils.generateWhere(queryStructReq, + queryStatement.getHeadlessModel().getDataDate()); log.info("in generateSqlCommand, complete where:{}", where); metricTable.setWhere(where); metricTable.setAggOption(AggOption.AGGREGATION); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); - String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd), + String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructReq), metricTableName, - sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd), - sqlGenerateUtils.getLimit(queryStructCmd)); - if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) { + sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq), + sqlGenerateUtils.getLimit(queryStructReq)); + if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { sqlCommand.setSupportWith(false); - sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd), + sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructReq), metricTableName, - sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd), - sqlGenerateUtils.getLimit(queryStructCmd)); + sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq), + sqlGenerateUtils.getLimit(queryStructReq)); } sqlCommand.setSql(sql); return sqlCommand; @@ -122,12 +105,11 @@ public class CalculateAggConverter implements HeadlessConverter { } @Override - public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception { - QueryStructReq queryStructCmd = queryStatement.getQueryStructReq(); + public void convert(QueryStatement queryStatement) throws Exception { ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); - DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelIds().get(0)); - ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd, - EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); + DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp(); + ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, + EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setTables(parseSqlReq.getTables()); sqlCommend.setRootPath(parseSqlReq.getRootPath()); @@ -149,38 +131,39 @@ public class CalculateAggConverter implements HeadlessConverter { return false; } - public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum, + public ParseSqlReq generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, String version) throws Exception { - check(queryStructCmd); + QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); + check(queryStructReq); ParseSqlReq sqlCommand = new ParseSqlReq(); - sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds())); + sqlCommand.setRootPath(queryStructReq.getModelIdStr()); String metricTableName = "v_metric_tb_tmp"; MetricTable metricTable = new MetricTable(); metricTable.setAlias(metricTableName); - metricTable.setMetrics(queryStructCmd.getMetrics()); - metricTable.setDimensions(queryStructCmd.getGroups()); - String where = queryStructUtils.generateWhere(queryStructCmd); + metricTable.setMetrics(queryStructReq.getMetrics()); + metricTable.setDimensions(queryStructReq.getGroups()); + String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getHeadlessModel().getDataDate()); log.info("in generateSqlCommend, complete where:{}", where); metricTable.setWhere(where); metricTable.setAggOption(AggOption.AGGREGATION); sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); - boolean isOver = isOverRatio(queryStructCmd); + boolean isOver = isOverRatio(queryStructReq); String sql = ""; switch (engineTypeEnum) { case H2: - sql = new H2EngineSql().sql(queryStructCmd, isOver, true, metricTableName); + sql = new H2EngineSql().sql(queryStructReq, isOver, true, metricTableName); break; case MYSQL: case DORIS: case CLICKHOUSE: - if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) { + if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { sqlCommand.setSupportWith(false); } if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) { - sql = new MysqlEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName); + sql = new MysqlEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName); } else { - sql = new CkEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName); + sql = new CkEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName); } break; default: diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java new file mode 100644 index 000000000..e3f71d1dc --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java @@ -0,0 +1,82 @@ +package com.tencent.supersonic.headless.core.parser.converter; + +import com.tencent.supersonic.common.pojo.ColumnOrder; +import com.tencent.supersonic.headless.api.pojo.Param; +import com.tencent.supersonic.headless.api.request.MetricQueryReq; +import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.core.parser.HeadlessConverter; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Component("ParserDefaultConverter") +@Slf4j +public class ParserDefaultConverter implements HeadlessConverter { + + private final SqlGenerateUtils sqlGenerateUtils; + + private final CalculateAggConverter calculateConverterAgg; + + public ParserDefaultConverter(CalculateAggConverter calculateConverterAgg, + SqlGenerateUtils sqlGenerateUtils) { + this.calculateConverterAgg = calculateConverterAgg; + this.sqlGenerateUtils = sqlGenerateUtils; + } + + @Override + public boolean accept(QueryStatement queryStatement) { + if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) { + return false; + } + return !calculateConverterAgg.accept(queryStatement); + } + + @Override + public void convert(QueryStatement queryStatement) throws Exception { + QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); + MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); + MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement); + queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, + queryStatement.getHeadlessModel().getDataDate())); + BeanUtils.copyProperties(metricReq, metricQueryReq); + } + + public MetricQueryReq generateSqlCommand(QueryStructReq queryStructReq, QueryStatement queryStatement) { + MetricQueryReq metricQueryReq = new MetricQueryReq(); + metricQueryReq.setMetrics(queryStructReq.getMetrics()); + metricQueryReq.setDimensions(queryStructReq.getGroups()); + String where = sqlGenerateUtils.generateWhere(queryStructReq, + queryStatement.getHeadlessModel().getDataDate()); + log.info("in generateSqlCommend, complete where:{}", where); + + metricQueryReq.setWhere(where); + metricQueryReq.setOrder(queryStructReq.getOrders().stream() + .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList())); + metricQueryReq.setVariables(queryStructReq.getParams().stream() + .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); + metricQueryReq.setLimit(queryStructReq.getLimit()); + String rootPath = queryStructReq.getModelIdStr(); + metricQueryReq.setRootPath(rootPath); + + // support detail query + if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) { + Map dataSourceMap = queryStatement.getHeadlessModel().getModelMap(); + for (Long modelId : queryStructReq.getModelIds()) { + String modelBizName = dataSourceMap.get(modelId).getName(); + String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName); + metricQueryReq.getMetrics().add(internalMetricName); + } + } + + return metricQueryReq; + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ConnectInfo.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java similarity index 76% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ConnectInfo.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java index 9a3cc7063..305865e47 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ConnectInfo.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo; +package com.tencent.supersonic.headless.core.pojo; import lombok.Data; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/Database.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java similarity index 91% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/Database.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java index de0f95993..511785c83 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/Database.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo; +package com.tencent.supersonic.headless.core.pojo; import com.google.common.collect.Lists; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/JdbcDataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java similarity index 98% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/JdbcDataSource.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java index 6d2555a1f..27ff13e45 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/JdbcDataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo; +package com.tencent.supersonic.headless.core.pojo; import com.alibaba.druid.filter.Filter; import com.alibaba.druid.pool.DruidDataSource; @@ -6,7 +6,7 @@ import com.alibaba.druid.wall.WallConfig; import com.alibaba.druid.wall.WallFilter; import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.headless.server.utils.JdbcDataSourceUtils; +import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/ParserSvrResponse.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ParserSvrResponse.java similarity index 88% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/ParserSvrResponse.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ParserSvrResponse.java index 32eb8ce4f..24e16a8d0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/ParserSvrResponse.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ParserSvrResponse.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.persistence.pojo; +package com.tencent.supersonic.headless.core.pojo; public class ParserSvrResponse { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java similarity index 86% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/QueryStatement.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index e853b86ca..6bb5fb4bf 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -1,13 +1,14 @@ -package com.tencent.supersonic.headless.core.persistence.pojo; +package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import lombok.Data; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; import java.util.List; -import org.apache.commons.lang3.tuple.Triple; @Data public class QueryStatement { @@ -26,6 +27,8 @@ public class QueryStatement { private Boolean enableOptimize = true; private Triple minMaxTime; + private HeadlessModel headlessModel; + public boolean isOk() { this.ok = "".equals(errMsg) && !"".equals(sql); return ok; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java similarity index 85% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java index e03233cb0..609508803 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import com.tencent.supersonic.headless.api.enums.ModelSourceType; import lombok.Data; @@ -9,6 +9,8 @@ import java.util.List; @Data public class DataModelYamlTpl { + private Long id; + private String name; private Long sourceId; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionTimeTypeParamsTpl.java similarity index 71% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionTimeTypeParamsTpl.java index 7d4f5084a..39ccea7c1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionTimeTypeParamsTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.Data; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java similarity index 84% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java index a9dd60f66..f8fc8937a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/IdentifyYamlTpl.java similarity index 82% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/IdentifyYamlTpl.java index cd612cf04..5e2744676 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/IdentifyYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MeasureYamlTpl.java similarity index 85% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MeasureYamlTpl.java index aa797d5c9..e2088f8f2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MeasureYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java similarity index 73% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java index c75eaaee5..daf314e67 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.Data; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricYamlTpl.java similarity index 79% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricYamlTpl.java index d60038ef2..41c5dc0a3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricYamlTpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; +package com.tencent.supersonic.headless.core.pojo.yaml; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 1f6f55fa1..fb4288469 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -1,18 +1,15 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.executor.JdbcExecutor; +import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.parser.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser; -import com.tencent.supersonic.headless.core.parser.convert.DefaultDimValueConverter; -import com.tencent.supersonic.headless.core.parser.convert.ZipperModelConverter; -import com.tencent.supersonic.headless.core.executor.JdbcExecutor; -import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.optimizer.DetailQuery; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; -import com.tencent.supersonic.headless.core.parser.convert.CalculateAggConverter; -import com.tencent.supersonic.headless.core.parser.convert.MetricCheckConverter; -import com.tencent.supersonic.headless.core.parser.convert.ParserDefaultConverter; +import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter; +import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -22,14 +19,14 @@ import java.util.stream.Collectors; public class ComponentFactory { private static List headlessConverters = new ArrayList<>(); - private static List queryExecutors = new ArrayList<>(); private static Map queryOptimizers = new HashMap<>(); + private static List queryExecutors = new ArrayList<>(); private static SqlParser sqlParser; static { initSemanticConverter(); - initQueryExecutors(); initQueryOptimizer(); + initQueryExecutors(); } public static List getSemanticConverters() { @@ -39,13 +36,6 @@ public class ComponentFactory { return headlessConverters; } - public static List getQueryExecutors() { - if (queryExecutors.isEmpty()) { - initQueryExecutors(); - } - return queryExecutors; - } - public static List getQueryOptimizers() { if (queryOptimizers.isEmpty()) { initQueryOptimizer(); @@ -53,6 +43,13 @@ public class ComponentFactory { return queryOptimizers.values().stream().collect(Collectors.toList()); } + public static List getQueryExecutors() { + if (queryExecutors.isEmpty()) { + initQueryExecutors(); + } + return queryExecutors; + } + public static SqlParser getSqlParser() { if (sqlParser == null) { sqlParser = ContextUtils.getContext().getBean("CalciteSqlParser", CalciteSqlParser.class); @@ -76,16 +73,13 @@ public class ComponentFactory { queryOptimizers.put("DetailQuery", getBean("DetailQuery", DetailQuery.class)); } - private static void initSemanticConverter() { - headlessConverters.add(getBean("MetricCheckConverter", MetricCheckConverter.class)); - headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); - headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); - headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); - headlessConverters.add(getBean("ZipperModelConverter", ZipperModelConverter.class)); - } - private static void initQueryExecutors() { queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class)); } + private static void initSemanticConverter() { + headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); + headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/JdbcDataSourceUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java similarity index 97% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/JdbcDataSourceUtils.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java index 218511758..a7a06e4ba 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/JdbcDataSourceUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java @@ -1,4 +1,22 @@ -package com.tencent.supersonic.headless.server.utils; +package com.tencent.supersonic.headless.core.utils; + +import com.alibaba.druid.util.StringUtils; +import com.tencent.supersonic.common.util.MD5Util; +import com.tencent.supersonic.headless.api.enums.DataType; +import com.tencent.supersonic.headless.api.response.DatabaseResp; +import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; import static com.tencent.supersonic.common.pojo.Constants.COLON; @@ -9,23 +27,6 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR; import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE; import static com.tencent.supersonic.common.pojo.Constants.SPACE; -import com.alibaba.druid.util.StringUtils; -import com.tencent.supersonic.headless.api.enums.DataType; -import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.common.util.MD5Util; -import com.tencent.supersonic.headless.server.pojo.Database; -import com.tencent.supersonic.headless.server.pojo.JdbcDataSource; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; -import java.util.regex.Matcher; -import javax.sql.DataSource; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - @Slf4j public class JdbcDataSourceUtils { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 0f3e184af..8664c39ab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -1,21 +1,61 @@ package com.tencent.supersonic.headless.core.utils; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; - import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.DateModeUtils; +import com.tencent.supersonic.common.util.SqlFilterUtils; +import com.tencent.supersonic.common.util.StringUtil; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; + @Component @Slf4j public class SqlGenerateUtils { + private final SqlFilterUtils sqlFilterUtils; + + private final DateModeUtils dateModeUtils; + + @Value("${metricParser.agg.mysql.lowVersion:5.7}") + private String mysqlLowVersion; + @Value("${metricParser.agg.ck.lowVersion:20.4}") + private String ckLowVersion; + @Value("${internal.metric.cnt.suffix:internal_cnt}") + private String internalMetricNameSuffix; + + public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils, + DateModeUtils dateModeUtils) { + this.sqlFilterUtils = sqlFilterUtils; + this.dateModeUtils = dateModeUtils; + } + public static String getUnionSelect(QueryStructReq queryStructCmd) { StringBuilder sb = new StringBuilder(); int locate = 0; @@ -82,4 +122,112 @@ public class SqlGenerateUtils { .collect(Collectors.joining(",")); } + public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) { + String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter()); + String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp); + return mergeDateWhereClause(queryStructReq, whereClauseFromFilter, whereFromDate); + } + + private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter, + String whereFromDate) { + if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { + return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); + } else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { + return whereClauseFromFilter; + } else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) { + return whereFromDate; + } else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) { + log.info("the current date information is empty, enter the date initialization logic"); + return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo()); + } + return whereClauseFromFilter; + } + + private String getDateWhereClause(DateConf dateInfo, ItemDateResp dateDate) { + if (Objects.isNull(dateDate) + || Strings.isEmpty(dateDate.getStartDate()) + && Strings.isEmpty(dateDate.getEndDate())) { + if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) { + return dateModeUtils.listDateStr(dateDate, dateInfo); + } + if (dateInfo.getDateMode().equals(DateConf.DateMode.BETWEEN)) { + return dateModeUtils.betweenDateStr(dateDate, dateInfo); + } + if (dateModeUtils.hasAvailableDataMode(dateInfo)) { + return dateModeUtils.hasDataModeStr(dateDate, dateInfo); + } + + return dateModeUtils.defaultRecentDateInfo(dateInfo); + } + log.info("dateDate:{}", dateDate); + return dateModeUtils.getDateWhereStr(dateInfo, dateDate); + } + + public Triple getBeginEndTime(QueryStructReq queryStructCmd, ItemDateResp dataDate) { + if (Objects.isNull(queryStructCmd.getDateInfo())) { + return Triple.of("", "", ""); + } + DateConf dateConf = queryStructCmd.getDateInfo(); + String dateInfo = dateModeUtils.getSysDateCol(dateConf); + if (dateInfo.isEmpty()) { + return Triple.of("", "", ""); + } + switch (dateConf.getDateMode()) { + case AVAILABLE: + case BETWEEN: + return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate()); + case LIST: + return Triple.of(dateInfo, Collections.min(dateConf.getDateList()), + Collections.max(dateConf.getDateList())); + case RECENT: + LocalDate dateMax = LocalDate.now().minusDays(1); + LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1); + if (Objects.isNull(dataDate)) { + return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), + dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT))); + } + switch (dateConf.getPeriod()) { + case DAY: + ImmutablePair dayInfo = dateModeUtils.recentDay(dataDate, dateConf); + return Triple.of(dateInfo, dayInfo.left, dayInfo.right); + case WEEK: + ImmutablePair weekInfo = dateModeUtils.recentWeek(dataDate, dateConf); + return Triple.of(dateInfo, weekInfo.left, weekInfo.right); + case MONTH: + List> rets = dateModeUtils.recentMonth(dataDate, dateConf); + Optional minBegins = rets.stream().map(i -> i.left).sorted().findFirst(); + Optional maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder()) + .findFirst(); + if (minBegins.isPresent() && maxBegins.isPresent()) { + return Triple.of(dateInfo, minBegins.get(), maxBegins.get()); + } + break; + default: + break; + } + break; + default: + break; + + } + return Triple.of("", "", ""); + } + + public boolean isSupportWith(EngineType engineTypeEnum, String version) { + if (engineTypeEnum.equals(EngineType.MYSQL) && Objects.nonNull(version) && version.startsWith( + mysqlLowVersion)) { + return false; + } + if (engineTypeEnum.equals(EngineType.CLICKHOUSE) && Objects.nonNull(version) + && StringUtil.compareVersion(version, + ckLowVersion) < 0) { + return false; + } + return true; + } + + public String generateInternalMetricName(String modelBizName) { + return modelBizName + UNDERLINE + internalMetricNameSuffix; + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SqlUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java similarity index 98% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SqlUtils.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java index 2881f215c..a45a8bbf1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SqlUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.utils; +package com.tencent.supersonic.headless.core.utils; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; @@ -7,7 +7,7 @@ import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.server.pojo.JdbcDataSource; +import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import java.rmi.ServerException; import java.sql.Connection; import java.sql.ResultSet; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SysTimeDimensionBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java similarity index 96% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SysTimeDimensionBuilder.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java index 3f9d2942e..ff9509248 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SysTimeDimensionBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java @@ -1,14 +1,14 @@ -package com.tencent.supersonic.headless.server.utils; +package com.tencent.supersonic.headless.core.utils; -import com.tencent.supersonic.headless.api.enums.DimensionType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor; -import java.util.List; - +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import lombok.extern.slf4j.Slf4j; +import java.util.List; + @Slf4j public class SysTimeDimensionBuilder { diff --git a/headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/MaterializationQueryTest.java b/headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/MaterializationQueryTest.java deleted file mode 100644 index e44c18e6a..000000000 --- a/headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/MaterializationQueryTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.tencent.supersonic.headless.query.domain.calcite; - -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.DateConf.DateMode; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; -import com.tencent.supersonic.headless.core.parser.QueryParser; -import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import com.tencent.supersonic.headless.core.utils.QueryUtils; -import lombok.extern.slf4j.Slf4j; - -import java.util.Arrays; - -@Slf4j -public class MaterializationQueryTest { - - private final QueryParser queryParser; - private final QueryUtils queryUtils; - - public MaterializationQueryTest(QueryParser queryParser, - QueryUtils queryUtils) { - this.queryParser = queryParser; - this.queryUtils = queryUtils; - } - - public void test() { - QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setModelId(1L); - - Aggregator aggregator = new Aggregator(); - aggregator.setFunc(AggOperatorEnum.UNKNOWN); - aggregator.setColumn("pv"); - queryStructReq.setAggregators(Arrays.asList(aggregator)); - - queryStructReq.setGroups(Arrays.asList("department")); - DateConf dateConf = new DateConf(); - dateConf.setDateMode(DateMode.LIST); - dateConf.setDateList(Arrays.asList("2023-08-01")); - queryStructReq.setDateInfo(dateConf); - - try { - QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructReq); - queryStatement.setIsS2SQL(false); - queryStatement = queryParser.logicSql(queryStatement); - queryUtils.checkSqlParse(queryStatement); - queryStatement.setModelIds(queryStructReq.getModelIds()); - log.info("queryStatement:{}", queryStatement); - for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { - queryOptimizer.rewrite(queryStructReq, queryStatement); - } - //queryParser.test(queryStructReq,metricReq); - log.info("queryStatement:{}", queryStatement); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/headless/server/pom.xml b/headless/server/pom.xml index 349b8d72d..80c8fb19b 100644 --- a/headless/server/pom.xml +++ b/headless/server/pom.xml @@ -44,13 +44,7 @@ com.tencent.supersonic - headless-api - ${project.version} - compile - - - com.tencent.supersonic - common + headless-core ${project.version} @@ -96,6 +90,26 @@ h2 ${h2.version} + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + org.apache.poi + poi-ooxml-schemas + ${poi.version} + + + com.alibaba + easyexcel + ${easyexcel.version} + \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptorFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptorFactory.java deleted file mode 100644 index a92ede362..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptorFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.tencent.supersonic.headless.server.adaptor.db; - - -import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum; -import java.util.HashMap; -import java.util.Map; - - -public class DbAdaptorFactory { - - private static Map dbAdaptorMap; - - static { - dbAdaptorMap = new HashMap<>(); - dbAdaptorMap.put(EngineTypeEnum.CLICKHOUSE.getName(), new ClickHouseAdaptor()); - dbAdaptorMap.put(EngineTypeEnum.MYSQL.getName(), new MysqlAdaptor()); - dbAdaptorMap.put(EngineTypeEnum.H2.getName(), new H2Adaptor()); - } - - public static DbAdaptor getEngineAdaptor(String engineType) { - return dbAdaptorMap.get(engineType); - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/ApiHeaderCheck.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java similarity index 83% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/ApiHeaderCheck.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java index c299affd9..3b53782ff 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/ApiHeaderCheck.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.annotation; +package com.tencent.supersonic.headless.server.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/S2SQLDataPermission.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2SQLDataPermission.java similarity index 84% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/S2SQLDataPermission.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2SQLDataPermission.java index ebe6235b0..81df3dcb1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/S2SQLDataPermission.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2SQLDataPermission.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.annotation; +package com.tencent.supersonic.headless.server.annotation; import java.lang.annotation.Target; import java.lang.annotation.RetentionPolicy; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/StructDataPermission.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/StructDataPermission.java similarity index 83% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/StructDataPermission.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/StructDataPermission.java index e0e61af70..9744b8d99 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/StructDataPermission.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/StructDataPermission.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.annotation; +package com.tencent.supersonic.headless.server.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/ApiHeaderCheckAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java similarity index 94% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/ApiHeaderCheckAspect.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java index 1d1222770..c5c6bb871 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/ApiHeaderCheckAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.aspect; +package com.tencent.supersonic.headless.server.aspect; import com.tencent.supersonic.common.pojo.Pair; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; @@ -33,7 +33,7 @@ public class ApiHeaderCheckAspect { @Autowired private AppService appService; - @Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck)") + @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)") private void apiPermissionCheck() { } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/AuthCheckBaseAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java similarity index 99% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/AuthCheckBaseAspect.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java index 3bfba7c5f..b5f8dcb9e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/AuthCheckBaseAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.aspect; +package com.tencent.supersonic.headless.server.aspect; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/DimValueAspect.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 1b6a9c076..2723f7afc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.aspect; +package com.tencent.supersonic.headless.server.aspect; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.Filter; @@ -47,7 +47,7 @@ public class DimValueAspect { @Autowired private DimensionService dimensionService; - @Around("execution(* com.tencent.supersonic.headless.core.service.impl.QueryServiceImpl.queryBySql(..))") + @Around("execution(* com.tencent.supersonic.headless.server.service.impl.QueryServiceImpl.queryBySql(..))") public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapSqlEnable) { log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap"); @@ -133,9 +133,9 @@ public class DimValueAspect { filedNameToValueMap.put(fieldName, map); } - @Around("execution(* com.tencent.supersonic.headless.core.rest.QueryController.queryByStruct(..))" - + " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStruct(..))" - + " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStructWithAuth(..))") + @Around("execution(* com.tencent.supersonic.headless.server.rest.QueryController.queryByStruct(..))" + + " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStruct(..))" + + " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStructWithAuth(..))") public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapEnable) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/S2SQLDataAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java similarity index 97% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/S2SQLDataAspect.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java index 3e91cc18a..371251f3e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/S2SQLDataAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.aspect; +package com.tencent.supersonic.headless.server.aspect; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -56,7 +56,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { @Value("${permission.data.enable:true}") private Boolean permissionDataEnable; - @Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission)") + @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission)") private void s2SQLPermissionCheck() { } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/StructDataAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java similarity index 97% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/StructDataAspect.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java index 224a82eb0..59490df9b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/StructDataAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.aspect; +package com.tencent.supersonic.headless.server.aspect; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelService; @@ -50,7 +50,7 @@ public class StructDataAspect extends AuthCheckBaseAspect { @Value("${permission.data.enable:true}") private Boolean permissionDataEnable; - @Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.StructDataPermission)") + @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.StructDataPermission)") public void dataPermissionAOP() { } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/HeadlessSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java similarity index 92% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/HeadlessSchemaManager.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java index 97df5f204..37657e588 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/HeadlessSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java @@ -1,17 +1,11 @@ -package com.tencent.supersonic.headless.core.parser.calcite; +package com.tencent.supersonic.headless.server.manager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; @@ -25,6 +19,13 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams; import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; @@ -75,12 +76,14 @@ public class HeadlessSchemaManager { List metricYamlTpls = new ArrayList<>(); Map modelIdName = new HashMap<>(); catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); + DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next()); + headlessModel.setDatabaseResp(databaseResp); List modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); if (!CollectionUtils.isEmpty(modelRelas)) { headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { - Map dataSourceMap = dataModelYamlTpls.stream().map(d -> getDatasource(d)) + Map dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource) .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); headlessModel.setDatasourceMap(dataSourceMap); } @@ -116,9 +119,10 @@ public class HeadlessSchemaManager { } public static DataSource getDatasource(final DataModelYamlTpl d) { - DataSource datasource = DataSource.builder().sourceId(d.getSourceId()).sqlQuery(d.getSqlQuery()) - .name(d.getName()).tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) - .measures(getMeasures(d.getMeasures())).dimensions(getDimensions(d.getDimensions())).build(); + DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId()) + .sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery()) + .identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures())) + .dimensions(getDimensions(d.getDimensions())).build(); datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); if (Objects.nonNull(d.getModelSourceTypeEnum())) { datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java deleted file mode 100644 index 386d811a2..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.headless.server.manager; - -import com.tencent.supersonic.headless.api.response.MetricResp; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.utils.MetricConverter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; - - -@Slf4j -@Service -public class MetricYamlManager { - - public static List convert2YamlObj(List metrics) { - - List metricYamlTpls = new ArrayList<>(); - for (MetricResp metric : metrics) { - MetricYamlTpl metricYamlTpl = MetricConverter.convert2MetricYamlTpl(metric); - metricYamlTpls.add(metricYamlTpl); - } - return metricYamlTpls; - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/mapper/StatMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java similarity index 83% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/mapper/StatMapper.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java index ee88bff18..3d8f58ca5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/mapper/StatMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.persistence.mapper; +package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.api.pojo.QueryStat; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java similarity index 86% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepository.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java index 63c6cb7ee..56a012284 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.persistence.repository; +package com.tencent.supersonic.headless.server.persistence.repository; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.request.ItemUseReq; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java similarity index 93% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepositoryImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index 5a9e2f900..f1eaa5609 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -1,12 +1,13 @@ -package com.tencent.supersonic.headless.core.persistence.repository; +package com.tencent.supersonic.headless.server.persistence.repository.impl; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.pojo.QueryStat; -import com.tencent.supersonic.headless.core.persistence.mapper.StatMapper; +import com.tencent.supersonic.headless.server.persistence.mapper.StatMapper; +import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/DataDownload.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java similarity index 73% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/DataDownload.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java index 60e6517a3..6825dc1e8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/DataDownload.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.persistence.pojo; +package com.tencent.supersonic.headless.server.pojo; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java similarity index 95% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/QueryController.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index 438fdd4e8..2177f2782 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -1,28 +1,28 @@ -package com.tencent.supersonic.headless.core.rest; +package com.tencent.supersonic.headless.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.headless.api.enums.QueryType; import com.tencent.supersonic.headless.api.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.request.DownloadStructReq; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; -import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; +import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.enums.QueryType; -import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.response.SqlParserResp; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.service.DownloadService; -import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine; -import com.tencent.supersonic.headless.core.service.QueryService; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.server.service.DownloadService; +import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.server.service.QueryService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/SchemaController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/SchemaController.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java index d6d5a30ad..08697d258 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/SchemaController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.rest; +package com.tencent.supersonic.headless.server.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -12,7 +12,7 @@ import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.core.service.SchemaService; +import com.tencent.supersonic.headless.server.service.SchemaService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java index f17e63196..df67d0b78 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java @@ -7,9 +7,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import java.util.List; @@ -22,8 +22,6 @@ public interface Catalog { DatabaseResp getDatabaseByModelId(Long modelId); - String getModelFullPath(List modelIds); - DimensionResp getDimension(String bizName, Long modelId); DimensionResp getDimension(Long id); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/DownloadService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java similarity index 90% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/DownloadService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java index bd32e518a..a9667d46a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/DownloadService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.service; +package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.request.BatchDownloadReq; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/HeadlessQueryEngine.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java similarity index 85% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/HeadlessQueryEngine.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java index 5e0b518ba..5dc3e16b8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/HeadlessQueryEngine.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java @@ -1,10 +1,10 @@ -package com.tencent.supersonic.headless.core.service; +package com.tencent.supersonic.headless.server.service; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.executor.QueryExecutor; public interface HeadlessQueryEngine { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index dd38b11e7..24e804490 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -11,9 +11,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.MeasureResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import java.util.List; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java similarity index 90% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/QueryService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index 5969eff89..8475aaf77 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -1,20 +1,20 @@ -package com.tencent.supersonic.headless.core.service; +package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.headless.api.request.QueryItemReq; -import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; -import com.tencent.supersonic.headless.api.response.ExplainResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; +import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.ExplainResp; +import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck; import javax.servlet.http.HttpServletRequest; import java.util.List; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java similarity index 95% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/SchemaService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 65877a939..668ea3da1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.service; +package com.tencent.supersonic.headless.server.service; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java index b9fd0fbd0..066c0a545 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java @@ -7,9 +7,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.DatabaseService; @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; @Slf4j @Component @@ -58,11 +57,6 @@ public class CatalogImpl implements Catalog { return modelService.getDatabaseByModelId(modelId); } - @Override - public String getModelFullPath(List modelIds) { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); - } - @Override public DimensionResp getDimension(String bizName, Long modelId) { return dimensionService.getDimension(bizName, modelId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index d51a6a80a..3d04add7b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -5,17 +5,17 @@ import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository; -import com.tencent.supersonic.headless.server.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; -import com.tencent.supersonic.headless.server.utils.JdbcDataSourceUtils; -import com.tencent.supersonic.headless.server.utils.SqlUtils; +import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; +import com.tencent.supersonic.headless.core.utils.SqlUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/DownloadServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index 94bcf1d8a..76c082efb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.service.impl; +package com.tencent.supersonic.headless.server.service.impl; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; @@ -12,7 +12,6 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.headless.core.persistence.pojo.DataDownload; import com.tencent.supersonic.headless.api.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.request.DownloadStructReq; import com.tencent.supersonic.headless.api.enums.SemanticType; @@ -23,10 +22,11 @@ import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.service.QueryService; import com.tencent.supersonic.headless.core.utils.DataTransformUtils; -import com.tencent.supersonic.headless.core.service.DownloadService; +import com.tencent.supersonic.headless.server.pojo.DataDownload; +import com.tencent.supersonic.headless.server.service.DownloadService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.QueryService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/HeadlessQueryEngineImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java similarity index 71% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/HeadlessQueryEngineImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java index 70ba82861..5c985263b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/HeadlessQueryEngineImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java @@ -1,17 +1,21 @@ -package com.tencent.supersonic.headless.core.service.impl; +package com.tencent.supersonic.headless.server.service.impl; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.QueryParser; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import com.tencent.supersonic.headless.core.utils.QueryUtils; +import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; +import com.tencent.supersonic.headless.server.utils.QueryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -23,19 +27,24 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { private final QueryParser queryParser; private final Catalog catalog; private final QueryUtils queryUtils; + private final QueryStructUtils queryStructUtils; + private final HeadlessSchemaManager headlessSchemaManager; public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog, - QueryUtils queryUtils) { + QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager, + QueryStructUtils queryStructUtils) { this.queryParser = queryParser; this.catalog = catalog; this.queryUtils = queryUtils; + this.headlessSchemaManager = headlessSchemaManager; + this.queryStructUtils = queryStructUtils; } public QueryResultWithSchemaResp execute(QueryStatement queryStatement) { QueryResultWithSchemaResp queryResultWithColumns = null; QueryExecutor queryExecutor = route(queryStatement); if (queryExecutor != null) { - queryResultWithColumns = queryExecutor.execute(catalog, queryStatement); + queryResultWithColumns = queryExecutor.execute(queryStatement); queryResultWithColumns.setSql(queryStatement.getSql()); if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds()); @@ -46,6 +55,10 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { public QueryStatement plan(QueryStatement queryStatement) throws Exception { queryStatement.setEnableOptimize(queryUtils.enableOptimize()); + HeadlessModel headlessModel = headlessSchemaManager.get(queryStatement.getQueryStructReq().getModelIdStr()); + ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStatement.getQueryStructReq()); + headlessModel.setDataDate(itemDateResp); + queryStatement.setHeadlessModel(headlessModel); queryStatement = queryParser.logicSql(queryStatement); queryUtils.checkSqlParse(queryStatement); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); @@ -70,7 +83,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { } @Override - public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) { QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setParseSqlReq(sqlCommend); @@ -78,7 +91,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement)); } - public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) { QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setMetricReq(metricCommand); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index 732aa27fa..41a1ba0d0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -30,12 +30,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.manager.DimensionYamlManager; -import com.tencent.supersonic.headless.server.manager.MetricYamlManager; -import com.tencent.supersonic.headless.server.manager.ModelYamlManager; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.core.manager.DimensionYamlManager; +import com.tencent.supersonic.headless.core.manager.MetricYamlManager; +import com.tencent.supersonic.headless.core.manager.ModelYamlManager; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java similarity index 94% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/QueryServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 5a2b3e415..dda20b71c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.service.impl; +package com.tencent.supersonic.headless.server.service.impl; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; @@ -14,44 +14,44 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; -import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; +import com.tencent.supersonic.headless.api.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.Cache; +import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Item; +import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq; -import com.tencent.supersonic.headless.api.request.QueryItemReq; +import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; +import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; -import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.Item; -import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.response.AppDetailResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ExplainResp; +import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; +import com.tencent.supersonic.headless.api.response.ItemUseResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck; -import com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission; -import com.tencent.supersonic.headless.core.annotation.StructDataPermission; -import com.tencent.supersonic.headless.core.aspect.ApiHeaderCheckAspect; +import com.tencent.supersonic.headless.server.utils.QueryReqConverter; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.utils.StatUtils; +import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck; +import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission; +import com.tencent.supersonic.headless.server.annotation.StructDataPermission; +import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.parser.convert.QueryReqConverter; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine; -import com.tencent.supersonic.headless.core.service.QueryService; -import com.tencent.supersonic.headless.core.service.SchemaService; -import com.tencent.supersonic.headless.core.utils.QueryUtils; -import com.tencent.supersonic.headless.core.utils.StatUtils; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.server.utils.QueryUtils; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -125,13 +125,13 @@ public class QueryServiceImpl implements QueryService { return headlessQueryEngine.execute(queryStatement); } - private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception { + private QueryStatement convertToQueryStatement(QueryS2SQLReq queryS2SQLReq, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(querySqlCmd.getModelIds()); + filter.setModelIds(queryS2SQLReq.getModelIds()); SchemaService schemaService = ContextUtils.getBean(SchemaService.class); List modelSchemaResps = schemaService.fetchModelSchema(filter, user); - QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, modelSchemaResps); - queryStatement.setModelIds(querySqlCmd.getModelIds()); + QueryStatement queryStatement = queryReqConverter.convert(queryS2SQLReq, modelSchemaResps); + queryStatement.setModelIds(queryS2SQLReq.getModelIds()); return queryStatement; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/SchemaServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index 06f4f7c24..df4051fd8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.service.impl; +package com.tencent.supersonic.headless.server.service.impl; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -16,12 +16,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.core.service.QueryService; -import com.tencent.supersonic.headless.core.service.SchemaService; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java index 7fe80a158..25a7a1777 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java @@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; -import com.tencent.supersonic.headless.server.pojo.ConnectInfo; -import com.tencent.supersonic.headless.server.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import com.tencent.supersonic.headless.core.pojo.Database; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import java.util.Arrays; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 8113c84b2..e1462c6f7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -9,7 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.request.DimensionReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; @@ -76,12 +75,4 @@ public class DimensionConverter { return dimensionResp; } - public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) { - DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl(); - BeanUtils.copyProperties(dimension, dimensionYamlTpl); - dimensionYamlTpl.setName(dimension.getBizName()); - dimensionYamlTpl.setOwners(dimension.getCreatedBy()); - return dimensionYamlTpl; - } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index d42bbb9e1..37747094d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -1,25 +1,19 @@ package com.tencent.supersonic.headless.server.utils; import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import org.springframework.beans.BeanUtils; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; public class MetricConverter { @@ -50,12 +44,6 @@ public class MetricConverter { return metricDO; } - public static MeasureYamlTpl convert(Measure measure) { - MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl(); - measureYamlTpl.setName(measure.getBizName()); - return measureYamlTpl; - } - public static MetricResp convert2MetricResp(MetricDO metricDO, Map modelMap, List collect) { MetricResp metricResp = new MetricResp(); BeanUtils.copyProperties(metricDO, metricResp); @@ -80,19 +68,4 @@ public class MetricConverter { return metricResp; } - public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) { - MetricYamlTpl metricYamlTpl = new MetricYamlTpl(); - BeanUtils.copyProperties(metric, metricYamlTpl); - metricYamlTpl.setName(metric.getBizName()); - metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); - MetricTypeParams exprMetricTypeParams = metric.getTypeParams(); - MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); - metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr()); - List measures = exprMetricTypeParams.getMeasures(); - metricTypeParamsYamlTpl.setMeasures( - measures.stream().map(MetricConverter::convert).collect(Collectors.toList())); - metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl); - return metricYamlTpl; - } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java similarity index 79% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/QueryReqConverter.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 053d01523..23e6b146d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.parser.convert; +package com.tencent.supersonic.headless.server.utils; import com.tencent.supersonic.common.pojo.Aggregator; @@ -10,22 +10,21 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.enums.AggOption; +import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.MetricTable; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine; -import com.tencent.supersonic.headless.core.utils.QueryStructUtils; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.server.service.Catalog; -import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -46,37 +45,40 @@ import java.util.stream.Stream; @Component @Slf4j public class QueryReqConverter { - @Autowired - private ModelService domainService; - @Autowired - private HeadlessQueryEngine parserService; + private HeadlessQueryEngine headlessQueryEngine; @Autowired private QueryStructUtils queryStructUtils; + @Autowired + private SqlGenerateUtils sqlGenerateUtils; + @Autowired private Catalog catalog; - public QueryStatement convert(QueryS2SQLReq databaseReq, List modelSchemaResps) throws Exception { + public QueryStatement convert(QueryS2SQLReq queryS2SQLReq, + List modelSchemaResps) throws Exception { if (CollectionUtils.isEmpty(modelSchemaResps)) { return new QueryStatement(); } + Map modelSchemaRespMap = modelSchemaResps.stream() + .collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp)); //1.convert name to bizName - convertNameToBizName(databaseReq, modelSchemaResps); + convertNameToBizName(queryS2SQLReq, modelSchemaResps); //2.functionName corrector - functionNameCorrector(databaseReq); + functionNameCorrector(queryS2SQLReq); //3.correct tableName - correctTableName(databaseReq); + correctTableName(queryS2SQLReq); - String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql()); + String tableName = SqlParserSelectHelper.getTableName(queryS2SQLReq.getSql()); if (StringUtils.isEmpty(tableName)) { return new QueryStatement(); } //4.build MetricTables - List allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql()); + List allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql()); List metrics = getMetrics(modelSchemaResps, allFields); - QueryStructReq queryStructCmd = new QueryStructReq(); + QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); metricTable.setMetrics(metrics); @@ -88,42 +90,43 @@ public class QueryReqConverter { // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { metricTable.setMetrics(new ArrayList<>()); - for (Long modelId : databaseReq.getModelIds()) { - metricTable.getMetrics().add(queryStructUtils.generateInternalMetricName(modelId, - metricTable.getDimensions())); + for (Long modelId : queryS2SQLReq.getModelIds()) { + ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId); + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName())); } } else { - queryStructCmd.setAggregators( + queryStructReq.setAggregators( metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( Collectors.toList())); } - AggOption aggOption = getAggOption(databaseReq); + AggOption aggOption = getAggOption(queryS2SQLReq); metricTable.setAggOption(aggOption); List tables = new ArrayList<>(); tables.add(metricTable); //4.build ParseSqlReq ParseSqlReq result = new ParseSqlReq(); - BeanUtils.copyProperties(databaseReq, result); + BeanUtils.copyProperties(queryS2SQLReq, result); - result.setRootPath(catalog.getModelFullPath(databaseReq.getModelIds())); + result.setRootPath(queryS2SQLReq.getModelIdStr()); result.setTables(tables); - DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0)); - if (!queryStructUtils.isSupportWith(EngineTypeEnum.valueOf(database.getType().toUpperCase()), + DatabaseResp database = catalog.getDatabaseByModelId(queryS2SQLReq.getModelIds().get(0)); + if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); } //5.physicalSql by ParseSqlReq - queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql())); - queryStructCmd.setModelIds(databaseReq.getModelIds().stream().collect(Collectors.toSet())); - queryStructCmd.setQueryType(getQueryType(aggOption)); - log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd); + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql())); + queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds())); + queryStructReq.setQueryType(getQueryType(aggOption)); + log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructCmd); + queryStatement.setQueryStructReq(queryStructReq); queryStatement.setParseSqlReq(result); queryStatement.setIsS2SQL(true); - queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd)); - queryStatement = parserService.plan(queryStatement); + queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); + queryStatement.setModelIds(queryS2SQLReq.getModelIds()); + queryStatement = headlessQueryEngine.plan(queryStatement); queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql())); return queryStatement; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java similarity index 59% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryStructUtils.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 567534d37..2f9ba1940 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.utils; +package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -9,32 +9,25 @@ import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; -import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.headless.api.request.ParseSqlReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; -import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; +import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.core.service.SchemaService; -import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.server.service.SchemaService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Triple; -import org.apache.logging.log4j.util.Strings; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -55,7 +48,6 @@ import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.DAY; import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; import static com.tencent.supersonic.common.pojo.Constants.WEEK; @@ -75,12 +67,6 @@ public class QueryStructUtils { private final DateModeUtils dateModeUtils; private final SqlFilterUtils sqlFilterUtils; private final Catalog catalog; - @Value("${internal.metric.cnt.suffix:internal_cnt}") - private String internalMetricNameSuffix; - @Value("${metricParser.agg.mysql.lowVersion:5.7}") - private String mysqlLowVersion; - @Value("${metricParser.agg.ck.lowVersion:20.4}") - private String ckLowVersion; @Autowired private SchemaService schemaService; @@ -113,7 +99,6 @@ public class QueryStructUtils { dimensionIds.add(pair.get(col).get(0).getId()); } } - return dimensionIds; } @@ -136,63 +121,6 @@ public class QueryStructUtils { return metricIds; } - public String getDateWhereClause(QueryStructReq queryStructCmd) { - DateConf dateInfo = queryStructCmd.getDateInfo(); - if (Objects.isNull(dateInfo) || Objects.isNull(dateInfo.getDateMode())) { - return ""; - } - if (dateInfo.getDateMode().equals(DateMode.RECENT)) { - if (dateInfo.getUnit() <= 0) { - return ""; - } - } - - List dimensionIds = getDimensionIds(queryStructCmd); - List metricIds = getMetricIds(queryStructCmd); - - ItemDateResp dateDate = catalog.getItemDate( - new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()), - new ItemDateFilter(metricIds, TypeEnums.METRIC.getName())); - if (Objects.isNull(dateDate) - || Strings.isEmpty(dateDate.getStartDate()) - && Strings.isEmpty(dateDate.getEndDate())) { - if (dateInfo.getDateMode().equals(DateMode.LIST)) { - return dateModeUtils.listDateStr(dateDate, dateInfo); - } - if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) { - return dateModeUtils.betweenDateStr(dateDate, dateInfo); - } - if (dateModeUtils.hasAvailableDataMode(dateInfo)) { - return dateModeUtils.hasDataModeStr(dateDate, dateInfo); - } - - return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo()); - } - log.info("dateDate:{}", dateDate); - return dateModeUtils.getDateWhereStr(dateInfo, dateDate); - } - - public String generateWhere(QueryStructReq queryStructCmd) { - String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter()); - String whereFromDate = getDateWhereClause(queryStructCmd); - return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate); - } - - public String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter, - String whereFromDate) { - if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { - return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); - } else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { - return whereClauseFromFilter; - } else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) { - return whereFromDate; - } else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) { - log.info("the current date information is empty, enter the date initialization logic"); - return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo()); - } - return whereClauseFromFilter; - } - public Set getResNameEn(QueryStructReq queryStructCmd) { Set resNameEnSet = new HashSet<>(); queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn())); @@ -254,76 +182,13 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public String generateInternalMetricName(Long modelId, List groups) { - String internalMetricNamePrefix = ""; - MetaFilter modelFilter = new MetaFilter(Collections.singletonList(modelId)); - List dimensions = catalog.getDimensions(modelFilter); - if (!CollectionUtils.isEmpty(dimensions)) { - internalMetricNamePrefix = dimensions.get(0).getModelBizName(); - } - return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix; - } - - public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) { - if (engineTypeEnum.equals(EngineTypeEnum.MYSQL) && Objects.nonNull(version) && version.startsWith( - mysqlLowVersion)) { - return false; - } - if (engineTypeEnum.equals(EngineTypeEnum.CLICKHOUSE) && Objects.nonNull(version) - && StringUtil.compareVersion(version, - ckLowVersion) < 0) { - return false; - } - return true; - } - - public String generateZipperWhere(QueryStructReq queryStructCmd) { - String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter()); - String whereFromDate = getZipperDateWhereClause(queryStructCmd); - return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate); - } - - public String generateZipperWhere(QueryStructReq queryStructCmd, ParseSqlReq parseSqlReq) { - if (Objects.nonNull(parseSqlReq.getSql()) && !CollectionUtils.isEmpty(parseSqlReq.getTables()) - && Objects.nonNull(queryStructCmd.getDateInfo())) { - String sql = SqlParserRemoveHelper.removeWhere(parseSqlReq.getSql(), - dateModeUtils.getDateCol()); - parseSqlReq.getTables().stream().forEach(t -> { - if (Objects.nonNull(t)) { - List dimensions = new ArrayList<>(); - if (!CollectionUtils.isEmpty(t.getDimensions())) { - dimensions.addAll(t.getDimensions().stream() - .filter(d -> !dateModeUtils.getDateCol().contains(d.toLowerCase())).collect( - Collectors.toList())); - } - dimensions.add(dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo())); - dimensions.add(dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo())); - t.setDimensions(dimensions); - } - }); - return SqlParserAddHelper.addWhere(sql, - SqlParserSelectHelper.getTimeFilter(getTimeRanges(queryStructCmd), - dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo()), - dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo()))); - } - return parseSqlReq.getSql(); - } - - public String getZipperDateWhereClause(QueryStructReq queryStructCmd) { - List> timeRanges = getTimeRanges(queryStructCmd); - List wheres = new ArrayList<>(); - if (!CollectionUtils.isEmpty(timeRanges)) { - for (ImmutablePair range : timeRanges) { - String strWhere = dateModeUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range); - if (!strWhere.isEmpty()) { - wheres.add(strWhere); - } - } - if (!wheres.isEmpty()) { - return wheres.stream().collect(Collectors.joining(" or ", "(", ")")); - } - } - return ""; + public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) { + List dimensionIds = getDimensionIds(queryStructCmd); + List metricIds = getMetricIds(queryStructCmd); + ItemDateResp dateDate = catalog.getItemDate( + new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()), + new ItemDateFilter(metricIds, TypeEnums.METRIC.getName())); + return dateDate; } public Triple getBeginEndTime(QueryStructReq queryStructCmd) { @@ -377,61 +242,6 @@ public class QueryStructUtils { return Triple.of("", "", ""); } - public List> getTimeRanges(QueryStructReq queryStructCmd) { - List> ret = new ArrayList<>(); - if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStructCmd.getDateInfo())) { - return ret; - } - DateConf dateConf = queryStructCmd.getDateInfo(); - switch (dateConf.getDateMode()) { - case AVAILABLE: - case BETWEEN: - ret.add(ImmutablePair.of(dateConf.getStartDate(), dateConf.getEndDate())); - break; - case LIST: - for (String date : dateConf.getDateList()) { - ret.add(ImmutablePair.of(date, date)); - } - break; - case RECENT: - ItemDateResp dateDate = getItemDateResp(queryStructCmd); - LocalDate dateMax = LocalDate.now().minusDays(1); - LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1); - if (Objects.isNull(dateDate)) { - ret.add(ImmutablePair.of(dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)), - dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT)))); - break; - } - switch (dateConf.getPeriod()) { - case DAY: - ret.add(dateModeUtils.recentDay(dateDate, dateConf)); - break; - case WEEK: - ret.add(dateModeUtils.recentWeek(dateDate, dateConf)); - break; - case MONTH: - List> rets = dateModeUtils.recentMonth(dateDate, dateConf); - ret.addAll(rets); - break; - default: - break; - } - break; - default: - break; - } - return ret; - } - - private ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) { - List dimensionIds = getDimensionIds(queryStructCmd); - List metricIds = getMetricIds(queryStructCmd); - ItemDateResp dateDate = catalog.getItemDate( - new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()), - new ItemDateFilter(metricIds, TypeEnums.METRIC.getName())); - return dateDate; - } - public DateConf getDateConfBySql(String sql) { List fieldExpressions = SqlParserSelectHelper.getFilterExpression(sql); if (!CollectionUtils.isEmpty(fieldExpressions)) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryUtils.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index ef6cfa9c5..9e3e77c44 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.utils; +package com.tencent.supersonic.headless.server.utils; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; @@ -10,7 +10,8 @@ import com.tencent.supersonic.headless.api.enums.SemanticType; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; import lombok.extern.slf4j.Slf4j; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/StatUtils.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 3bb069418..45c379bd3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.utils; +package com.tencent.supersonic.headless.server.utils; import com.alibaba.ttl.TransmittableThreadLocal; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,17 +7,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.headless.api.enums.QueryOptMode; +import com.tencent.supersonic.headless.api.enums.QueryType; +import com.tencent.supersonic.headless.api.enums.QueryTypeBack; +import com.tencent.supersonic.headless.api.pojo.QueryStat; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.enums.QueryOptMode; -import com.tencent.supersonic.headless.api.enums.QueryTypeBack; -import com.tencent.supersonic.headless.api.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.QueryStat; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.core.persistence.repository.StatRepository; +import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.service.ModelService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; diff --git a/headless/core/src/main/resources/mapper/StatMapper.xml b/headless/server/src/main/resources/mapper/StatMapper.xml similarity index 97% rename from headless/core/src/main/resources/mapper/StatMapper.xml rename to headless/server/src/main/resources/mapper/StatMapper.xml index 4c5d48105..57bc055c0 100644 --- a/headless/core/src/main/resources/mapper/StatMapper.xml +++ b/headless/server/src/main/resources/mapper/StatMapper.xml @@ -2,7 +2,7 @@ - + diff --git a/headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java similarity index 92% rename from headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/HeadlessParserServiceTest.java rename to headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index bee42d294..f112d396b 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -1,26 +1,27 @@ -package com.tencent.supersonic.headless.query.domain.calcite; +package com.tencent.supersonic.headless.server.calcite; import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.headless.api.response.SqlParserResp; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.request.MetricQueryReq; -import com.tencent.supersonic.headless.core.parser.calcite.HeadlessSchemaManager; +import com.tencent.supersonic.headless.api.response.SqlParserResp; import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner; -import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement; import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.extern.slf4j.Slf4j; @Slf4j class HeadlessParserServiceTest { @@ -162,7 +163,7 @@ class HeadlessParserServiceTest { metric2.setTypeParams(metricTypeParams1); metric.add(metric2); - HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); + //HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); MetricQueryReq metricCommand = new MetricQueryReq(); metricCommand.setRootPath("s2"); @@ -250,6 +251,5 @@ class HeadlessParserServiceTest { headlessSchema.getDimension() .put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); - } } diff --git a/headless/core/src/test/java/com/tencent/supersonic/headless/query/service/DownloadServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java similarity index 95% rename from headless/core/src/test/java/com/tencent/supersonic/headless/query/service/DownloadServiceImplTest.java rename to headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java index e13576bd4..6574e57e9 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/headless/query/service/DownloadServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java @@ -1,20 +1,18 @@ -package com.tencent.supersonic.headless.query.service; +package com.tencent.supersonic.headless.server.service; import com.alibaba.excel.util.FileUtils; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.headless.api.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.RelateDimension; +import com.tencent.supersonic.headless.api.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.service.QueryService; -import com.tencent.supersonic.headless.core.service.impl.DownloadServiceImpl; -import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl; import org.junit.jupiter.api.Test; import org.mockito.Mockito; diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/model/application/ModelServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java similarity index 96% rename from headless/server/src/test/java/com/tencent/supersonic/headless/model/application/ModelServiceImplTest.java rename to headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java index 1b1368c19..3bc586c8f 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/model/application/ModelServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.model.application; +package com.tencent.supersonic.headless.server.service; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -17,12 +17,6 @@ import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; import com.tencent.supersonic.headless.server.persistence.repository.ModelRepository; -import com.tencent.supersonic.headless.server.service.DatabaseService; -import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.DomainService; -import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.ModelRelaService; -import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.impl.ModelServiceImpl; import com.tencent.supersonic.headless.server.utils.ModelConverter; import org.junit.jupiter.api.Assertions; diff --git a/headless/core/src/test/java/com/tencent/supersonic/headless/query/utils/DataTransformUtilsTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java similarity index 96% rename from headless/core/src/test/java/com/tencent/supersonic/headless/query/utils/DataTransformUtilsTest.java rename to headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java index 5689c2266..dfef206f9 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/headless/query/utils/DataTransformUtilsTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java @@ -1,10 +1,11 @@ -package com.tencent.supersonic.headless.query.utils; +package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.headless.core.utils.DataTransformUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List;