From c8fe6d2d0411f92630ef518efab621393de44354 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Tue, 6 Aug 2024 20:24:49 +0800 Subject: [PATCH] (improvement)(headless) support more db type (#1511) Co-authored-by: lxwcodemonkey --- .../headless/api/pojo/DBColumn.java | 19 ++++ .../headless/api/pojo/enums/EngineType.java | 3 +- headless/core/pom.xml | 5 + .../core/adaptor/db/BaseDbAdaptor.java | 72 ++++++++++++++ .../core/adaptor/db/ClickHouseAdaptor.java | 23 +---- .../headless/core/adaptor/db/DbAdaptor.java | 18 ++-- .../core/adaptor/db/DbAdaptorFactory.java | 1 + .../core/adaptor/db/DefaultDbAdaptor.java | 16 +++ .../headless/core/adaptor/db/H2Adaptor.java | 25 +---- .../core/adaptor/db/MysqlAdaptor.java | 20 +--- .../core/adaptor/db/PostgresqlAdaptor.java | 57 +++++++---- .../server/pojo/DbParameterFactory.java | 1 + .../server/pojo/OtherParametersBuilder.java | 36 +++++++ .../server/rest/DatabaseController.java | 32 +++--- .../server/service/DatabaseService.java | 11 ++- .../service/impl/DatabaseServiceImpl.java | 99 ++++++++++--------- .../server/utils/DatabaseConverter.java | 15 ++- 17 files changed, 296 insertions(+), 157 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java new file mode 100644 index 000000000..8e6d22efc --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DBColumn.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.headless.api.pojo; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DBColumn { + + private String columnName; + + private String dataType; + + private String comment; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java index 485fa181c..30f56fb25 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java @@ -9,7 +9,8 @@ public enum EngineType { CLICKHOUSE(3, "clickhouse"), KAFKA(4, "kafka"), H2(5, "h2"), - POSTGRESQL(6, "postgresql"); + POSTGRESQL(6, "postgresql"), + OTHER(7, "other"); private Integer code; diff --git a/headless/core/pom.xml b/headless/core/pom.xml index d8be4ffd8..2ca0bf6c3 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -116,6 +116,11 @@ duckdb_jdbc ${duckdb_jdbc.version} + + com.dameng + DmJdbcDriver18 + 8.1.2.192 + diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java new file mode 100644 index 000000000..b368f103c --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java @@ -0,0 +1,72 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +@Slf4j +public abstract class BaseDbAdaptor implements DbAdaptor { + + public List getDBs(ConnectInfo connectionInfo) throws SQLException { + List dbs = Lists.newArrayList(); + DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); + try { + ResultSet schemaSet = metaData.getSchemas(); + while (schemaSet.next()) { + String db = schemaSet.getString("TABLE_SCHEM"); + dbs.add(db); + } + } catch (Exception e) { + log.info("get meta schemas failed, try to get catalogs"); + } + try { + ResultSet catalogSet = metaData.getCatalogs(); + while (catalogSet.next()) { + String db = catalogSet.getString("TABLE_CAT"); + dbs.add(db); + } + } catch (Exception e) { + log.info("get meta catalogs failed, try to get schemas"); + } + return dbs; + } + + public List getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException { + List tables = Lists.newArrayList(); + DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); + ResultSet tableSet = metaData.getTables(schemaName, schemaName, null, new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString("TABLE_NAME"); + tables.add(tableName); + } + return tables; + } + + public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException { + List dbColumns = Lists.newArrayList(); + DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); + ResultSet columns = metaData.getColumns(schemaName, schemaName, tableName, null); + while (columns.next()) { + String columnName = columns.getString("COLUMN_NAME"); + String dataType = columns.getString("TYPE_NAME"); + String remarks = columns.getString("REMARKS"); + dbColumns.add(new DBColumn(columnName, dataType, remarks)); + } + return dbColumns; + } + + protected DatabaseMetaData getDatabaseMetaData(ConnectInfo connectionInfo) throws SQLException { + Connection connection = DriverManager.getConnection(connectionInfo.getUrl(), + connectionInfo.getUserName(), connectionInfo.getPassword()); + return connection.getMetaData(); + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java index 89f2591a2..fc153cb62 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java @@ -1,12 +1,13 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; + import java.util.HashMap; import java.util.Map; -public class ClickHouseAdaptor extends DbAdaptor { +public class ClickHouseAdaptor extends BaseDbAdaptor { @Override public String getDateFormat(String dateType, String dateFormat, String column) { @@ -30,19 +31,6 @@ public class ClickHouseAdaptor extends DbAdaptor { return column; } - @Override - public String getDbMetaQueryTpl() { - return " " - + " select " - + " name from system.databases " - + " where name not in('_temporary_and_external_tables','benchmark','default','system');"; - } - - @Override - public String getTableMetaQueryTpl() { - return "select name from system.tables where database = '%s';"; - } - @Override public String functionNameCorrector(String sql) { Map functionMap = new HashMap<>(); @@ -52,9 +40,4 @@ public class ClickHouseAdaptor extends DbAdaptor { return SqlReplaceHelper.replaceFunction(sql, functionMap); } - @Override - public String getColumnMetaQueryTpl() { - return "select name,type as dataType, comment from system.columns where database = '%s' and table='%s'"; - } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index 2715fd7bd..b81bb9ed6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -1,17 +1,23 @@ package com.tencent.supersonic.headless.core.adaptor.db; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import java.sql.SQLException; +import java.util.List; + /** * Adapters for different query engines to obtain table, field, and time formatting methods */ -public abstract class DbAdaptor { +public interface DbAdaptor { - public abstract String getDateFormat(String dateType, String dateFormat, String column); + String getDateFormat(String dateType, String dateFormat, String column); - public abstract String getColumnMetaQueryTpl(); + String functionNameCorrector(String sql); - public abstract String getDbMetaQueryTpl(); + List getDBs(ConnectInfo connectInfo) throws SQLException; - public abstract String getTableMetaQueryTpl(); + List getTables(ConnectInfo connectInfo, String schemaName) throws SQLException; + + List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException; - public abstract String functionNameCorrector(String sql); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java index 08c1f245d..15e62ffb9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java @@ -16,6 +16,7 @@ public class DbAdaptorFactory { dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor()); dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor()); dbAdaptorMap.put(EngineType.POSTGRESQL.getName(), new PostgresqlAdaptor()); + dbAdaptorMap.put(EngineType.OTHER.getName(), new DefaultDbAdaptor()); } public static DbAdaptor getEngineAdaptor(String engineType) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java new file mode 100644 index 000000000..18c434a06 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + + +public class DefaultDbAdaptor extends BaseDbAdaptor { + + @Override + public String getDateFormat(String dateType, String dateFormat, String column) { + return column; + } + + @Override + public String functionNameCorrector(String sql) { + return sql; + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java index 3fca71a74..687eb96d7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -public class H2Adaptor extends DbAdaptor { +public class H2Adaptor extends BaseDbAdaptor { @Override public String getDateFormat(String dateType, String dateFormat, String column) { @@ -27,30 +27,9 @@ public class H2Adaptor extends DbAdaptor { return column; } - @Override - public String getColumnMetaQueryTpl() { - return "SELECT COLUMN_NAME AS name, " - + " case DATA_TYPE" - + " when '12' then 'varchar'" - + " when '-5' then 'integer'" - + " when '8' then 'double'" - + " end AS dataType" - + " FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ='%s' AND TABLE_NAME = '%s'"; - } - - @Override - public String getDbMetaQueryTpl() { - return "SELECT DISTINCT TABLE_SCHEMA as name FROM INFORMATION_SCHEMA.TABLES WHERE STORAGE_TYPE = 'MEMORY'"; - } - - @Override - public String getTableMetaQueryTpl() { - return "SELECT TABLE_NAME as name FROM INFORMATION_SCHEMA.TABLES " - + "WHERE STORAGE_TYPE = 'MEMORY' AND TABLE_SCHEMA = '%s'"; - } - @Override public String functionNameCorrector(String sql) { return sql; } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java index 6d08905fa..b77cdb309 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java @@ -4,8 +4,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; -public class MysqlAdaptor extends DbAdaptor { - +public class MysqlAdaptor extends BaseDbAdaptor { /** * transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY) @@ -32,26 +31,9 @@ public class MysqlAdaptor extends DbAdaptor { return column; } - @Override - public String getDbMetaQueryTpl() { - return "select distinct TABLE_SCHEMA as name from information_schema.tables " - + "where TABLE_SCHEMA not in ('information_schema','mysql','performance_schema','sys');"; - } - - @Override - public String getTableMetaQueryTpl() { - return "select TABLE_NAME as name from information_schema.tables where TABLE_SCHEMA = '%s';"; - } - @Override public String functionNameCorrector(String sql) { return sql; } - @Override - public String getColumnMetaQueryTpl() { - return "SELECT COLUMN_NAME as name, DATA_TYPE as dataType, COLUMN_COMMENT as comment " - + "FROM information_schema.columns WHERE table_schema ='%s' AND table_name = '%s'"; - } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java index 4a108050a..f92c7d5d0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java @@ -1,16 +1,24 @@ package com.tencent.supersonic.headless.core.adaptor.db; +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.function.UnaryOperator; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; -public class PostgresqlAdaptor extends DbAdaptor { +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.UnaryOperator; + +public class PostgresqlAdaptor extends BaseDbAdaptor { @Override public String getDateFormat(String dateType, String dateFormat, String column) { @@ -34,17 +42,6 @@ public class PostgresqlAdaptor extends DbAdaptor { return column; } - @Override - public String getDbMetaQueryTpl() { - return " SELECT datname as name FROM pg_database WHERE datistemplate = false ;"; - } - - @Override - public String getTableMetaQueryTpl() { - return " SELECT table_name as name FROM information_schema.tables WHERE " - + "table_schema = 'public' AND table_catalog = '%s' ; "; - } - @Override public String functionNameCorrector(String sql) { Map functionMap = new HashMap<>(); @@ -79,10 +76,28 @@ public class PostgresqlAdaptor extends DbAdaptor { return SqlReplaceHelper.replaceFunction(sql, functionMap, functionCall); } - @Override - public String getColumnMetaQueryTpl() { - return " SELECT column_name as name, data_type as dataType FROM information_schema.columns " - + "WHERE table_schema = 'public' AND table_catalog = '%s' AND table_name = '%s' ; "; + public List getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException { + List tables = Lists.newArrayList(); + DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); + ResultSet tableSet = metaData.getTables(null, null, null, new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString("TABLE_NAME"); + tables.add(tableName); + } + return tables; + } + + public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException { + List dbColumns = Lists.newArrayList(); + DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); + ResultSet columns = metaData.getColumns(null, null, tableName, null); + while (columns.next()) { + String columnName = columns.getString("COLUMN_NAME"); + String dataType = columns.getString("TYPE_NAME"); + String remarks = columns.getString("REMARKS"); + dbColumns.add(new DBColumn(columnName, dataType, remarks)); + } + return dbColumns; } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java index ff40c818c..3d8b639b5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java @@ -15,6 +15,7 @@ public class DbParameterFactory { parametersBuilder.put(EngineType.CLICKHOUSE.getName(), new ClickHouseParametersBuilder()); parametersBuilder.put(EngineType.MYSQL.getName(), new MysqlParametersBuilder()); parametersBuilder.put(EngineType.POSTGRESQL.getName(), new PostgresqlParametersBuilder()); + parametersBuilder.put(EngineType.OTHER.getName(), new OtherParametersBuilder()); } public static DbParametersBuilder get(String engineType) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java new file mode 100644 index 000000000..4dd4cd79d --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java @@ -0,0 +1,36 @@ +package com.tencent.supersonic.headless.server.pojo; + + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +@Service +@Slf4j +public class OtherParametersBuilder implements DbParametersBuilder { + + @Override + public List build() { + List databaseParameters = new ArrayList<>(); + DatabaseParameter host = new DatabaseParameter(); + host.setComment("链接"); + host.setName("url"); + host.setPlaceholder("请输入链接"); + databaseParameters.add(host); + + DatabaseParameter userName = new DatabaseParameter(); + userName.setComment("用户名"); + userName.setName("username"); + userName.setPlaceholder("请输入用户名"); + databaseParameters.add(userName); + + DatabaseParameter password = new DatabaseParameter(); + password.setComment("密码"); + password.setName("password"); + password.setPlaceholder("请输入密码"); + databaseParameters.add(password); + return databaseParameters; + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 5470e83d5..5c5e62310 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; @@ -14,10 +15,12 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -76,22 +79,27 @@ public class DatabaseController { return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user); } - @RequestMapping("/getDbNames/{id}") - public SemanticQueryResp getDbNames(@PathVariable("id") Long id) { - return databaseService.getDbNames(id); + @RequestMapping("/getDbNames") + public List getDbNames(@RequestParam("id") Long databaseId) throws SQLException { + return databaseService.getDbNames(databaseId); } - @RequestMapping("/getTables/{id}/{db}") - public SemanticQueryResp getTables(@PathVariable("id") Long id, - @PathVariable("db") String db) { - return databaseService.getTables(id, db); + @RequestMapping("/getTables") + public List getTables(@RequestParam("databaseId") Long databaseId, + @RequestParam("db") String db) throws SQLException { + return databaseService.getTables(databaseId, db); } - @RequestMapping("/getColumns/{id}/{db}/{table}") - public SemanticQueryResp getColumns(@PathVariable("id") Long id, - @PathVariable("db") String db, - @PathVariable("table") String table) { - return databaseService.getColumns(id, db, table); + @RequestMapping("/getColumnsByName") + public List getColumnsByName(@RequestParam("databaseId") Long databaseId, @RequestParam("db") String db, + @RequestParam("table") String table) throws SQLException { + return databaseService.getColumns(databaseId, db, table); + } + + @RequestMapping("/getColumnsBySql") + public List getColumnsBySql(@RequestParam("databaseId") Long databaseId, + @RequestParam("sql") String sql) throws SQLException { + return databaseService.getColumns(databaseId, sql); } @GetMapping("/getDatabaseParameters") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index af0fe3a9d..6868775fa 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -1,11 +1,14 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; + +import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -30,9 +33,11 @@ public interface DatabaseService { void deleteDatabase(Long databaseId); - SemanticQueryResp getDbNames(Long id); + List getDbNames(Long id) throws SQLException; - SemanticQueryResp getTables(Long id, String db); + List getTables(Long id, String db) throws SQLException; - SemanticQueryResp getColumns(Long id, String db, String table); + List getColumns(Long id, String db, String table) throws SQLException; + + List getColumns(Long id, String sql) throws SQLException; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index d3ce5bbe5..bff8ea6a2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -3,10 +3,11 @@ package com.tencent.supersonic.headless.server.service.impl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Pair; import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; @@ -27,13 +28,16 @@ import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.util.TablesNamesFinder; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.HashMap; +import java.sql.SQLException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -134,9 +138,7 @@ public class DatabaseServiceImpl extends ServiceImpl dbTableName = getDbTableName(sql, databaseResp); - String db = dbTableName.first; - String table = dbTableName.second; - if (StringUtils.isBlank(db) || StringUtils.isBlank(table)) { - return; - } - SemanticQueryResp columnsWithComment = getColumns(databaseResp, db, table); - Map columnCommentMap = getColumnCommentMap(columnsWithComment.getResultList()); - List columns = semanticQueryResp.getColumns(); - for (QueryColumn column : columns) { - column.setComment(columnCommentMap.get(column.getNameEn())); - } - } - - private Map getColumnCommentMap(List> resultList) { - Map map = new HashMap<>(); - for (Map result : resultList) { - map.put(String.valueOf(result.get("name")), String.valueOf(result.get("comment"))); - } - return map; - } - private Pair getDbTableName(String sql, DatabaseResp databaseResp) { String dbTableName = SqlSelectHelper.getDbTableName(sql); if (StringUtils.isBlank(dbTableName)) { @@ -201,36 +179,63 @@ public class DatabaseServiceImpl extends ServiceImpl getDbNames(Long id) throws SQLException { DatabaseResp databaseResp = getDatabase(id); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl(); - return queryWithColumns(metaQueryTpl, DatabaseConverter.convert(databaseResp)); + DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); + return dbAdaptor.getDBs(DatabaseConverter.getConnectInfo(databaseResp)); } @Override - public SemanticQueryResp getTables(Long id, String db) { + public List getTables(Long id, String db) throws SQLException { DatabaseResp databaseResp = getDatabase(id); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl(); - String metaQuerySql = String.format(metaQueryTpl, db); - return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp)); + DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); + return dbAdaptor.getTables(DatabaseConverter.getConnectInfo(databaseResp), db); } @Override - public SemanticQueryResp getColumns(Long id, String db, String table) { + public List getColumns(Long id, String db, String table) throws SQLException { DatabaseResp databaseResp = getDatabase(id); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl(); - String metaQuerySql = String.format(metaQueryTpl, db, table); - return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp)); + return getColumns(databaseResp, db, table); } - public SemanticQueryResp getColumns(DatabaseResp databaseResp, String db, String table) { + public List getColumns(DatabaseResp databaseResp, String db, String table) throws SQLException { DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl(); - String metaQuerySql = String.format(metaQueryTpl, db, table); - return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp)); + return engineAdaptor.getColumns(DatabaseConverter.getConnectInfo(databaseResp), db, table); + } + + @Override + public List getColumns(Long id, String sql) throws SQLException { + String wrapSql = String.format("select * from (%s) a limit 1", sql); + DatabaseResp databaseResp = getDatabase(id); + SemanticQueryResp semanticQueryResp = executeSql(wrapSql, databaseResp); + List dbColumns = Lists.newArrayList(); + for (QueryColumn queryColumn : semanticQueryResp.getColumns()) { + DBColumn dbColumn = new DBColumn(); + dbColumn.setColumnName(queryColumn.getNameEn()); + dbColumn.setDataType(queryColumn.getType()); + dbColumns.add(dbColumn); + } + return dbColumns; + } + + public static void main(String[] args) { + try { + String sql = "SELECT * FROM mydatabase.mytable JOIN otherdatabase.othertable ON mytable.id = othertable.id"; + + // 解析SQL语句 + Statement statement = CCJSqlParserUtil.parse(sql); + + // 提取库表名 + TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); + List tableNames = tablesNamesFinder.getTableList(statement); + + // 打印库表名 + for (String tableName : tableNames) { + System.out.println("Table Name: " + tableName); + } + } catch (Exception e) { + e.printStackTrace(); + } } private void checkPermission(DatabaseResp databaseResp, User user) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java index 064880845..3fde4def5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java @@ -15,11 +15,7 @@ public class DatabaseConverter { public static Database convert(DatabaseResp databaseResp) { Database database = new Database(); BeanUtils.copyProperties(databaseResp, database); - ConnectInfo connectInfo = new ConnectInfo(); - connectInfo.setUserName(databaseResp.getUsername()); - connectInfo.setPassword(databaseResp.getPassword()); - connectInfo.setUrl(databaseResp.getUrl()); - connectInfo.setDatabase(databaseResp.getDatabase()); + ConnectInfo connectInfo = getConnectInfo(databaseResp); database.setConnectInfo(connectInfo); database.setVersion(databaseResp.getVersion()); return database; @@ -81,4 +77,13 @@ public class DatabaseConverter { return databaseResp; } + public static ConnectInfo getConnectInfo(DatabaseResp databaseResp) { + ConnectInfo connectInfo = new ConnectInfo(); + connectInfo.setUserName(databaseResp.getUsername()); + connectInfo.setPassword(databaseResp.getPassword()); + connectInfo.setUrl(databaseResp.getUrl()); + connectInfo.setDatabase(databaseResp.getDatabase()); + return connectInfo; + } + }