topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) support more db type (#1511)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-08-06 20:24:49 +08:00
committed by GitHub
parent 45fb83356f
commit c8fe6d2d04
17 changed files with 296 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
headless/core/pom.xml
View File

@@ -116,6 +116,11 @@
<artifactId>duckdb_jdbc</artifactId>
<version>${duckdb_jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.dameng</groupId>
<artifactId>DmJdbcDriver18</artifactId>
<version>8.1.2.192</version>
</dependency>
</dependencies>

72
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java Normal file
View File

@@ -0,0 +1,72 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import lombok.extern.slf4j.Slf4j;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
@Slf4j
public abstract class BaseDbAdaptor implements DbAdaptor {
public List<String> getDBs(ConnectInfo connectionInfo) throws SQLException {
List<String> dbs = Lists.newArrayList();
DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo);
try {
ResultSet schemaSet = metaData.getSchemas();
while (schemaSet.next()) {
String db = schemaSet.getString("TABLE_SCHEM");
dbs.add(db);
}
} catch (Exception e) {
log.info("get meta schemas failed, try to get catalogs");
}
try {
ResultSet catalogSet = metaData.getCatalogs();
while (catalogSet.next()) {
String db = catalogSet.getString("TABLE_CAT");
dbs.add(db);
}
} catch (Exception e) {
log.info("get meta catalogs failed, try to get schemas");
}
return dbs;
}
public List<String> getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException {
List<String> tables = Lists.newArrayList();
DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo);
ResultSet tableSet = metaData.getTables(schemaName, schemaName, null, new String[]{"TABLE"});
while (tableSet.next()) {
String tableName = tableSet.getString("TABLE_NAME");
tables.add(tableName);
}
return tables;
}
public List<DBColumn> getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException {
List<DBColumn> dbColumns = Lists.newArrayList();
DatabaseMetaData metaData = getDatabaseMetaData(connectInfo);
ResultSet columns = metaData.getColumns(schemaName, schemaName, tableName, null);
while (columns.next()) {
String columnName = columns.getString("COLUMN_NAME");
String dataType = columns.getString("TYPE_NAME");
String remarks = columns.getString("REMARKS");
dbColumns.add(new DBColumn(columnName, dataType, remarks));
}
return dbColumns;
}
protected DatabaseMetaData getDatabaseMetaData(ConnectInfo connectionInfo) throws SQLException {
Connection connection = DriverManager.getConnection(connectionInfo.getUrl(),
connectionInfo.getUserName(), connectionInfo.getPassword());
return connection.getMetaData();
}
}

23
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java
View File

@@ -1,12 +1,13 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import java.util.HashMap;
import java.util.Map;
public class ClickHouseAdaptor extends DbAdaptor {
public class ClickHouseAdaptor extends BaseDbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
@@ -30,19 +31,6 @@ public class ClickHouseAdaptor extends DbAdaptor {
return column;
}
@Override
public String getDbMetaQueryTpl() {
return " "
+ " select "
+ " name from system.databases "
+ " where name not in('_temporary_and_external_tables','benchmark','default','system');";
}
@Override
public String getTableMetaQueryTpl() {
return "select name from system.tables where database = '%s';";
}
@Override
public String functionNameCorrector(String sql) {
Map<String, String> functionMap = new HashMap<>();
@@ -52,9 +40,4 @@ public class ClickHouseAdaptor extends DbAdaptor {
return SqlReplaceHelper.replaceFunction(sql, functionMap);
}
@Override
public String getColumnMetaQueryTpl() {
return "select name,type as dataType, comment from system.columns where database = '%s' and table='%s'";
}
}

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java
View File

@@ -1,17 +1,23 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import java.sql.SQLException;
import java.util.List;
/**
* Adapters for different query engines to obtain table, field, and time formatting methods
*/
public abstract class DbAdaptor {
public interface DbAdaptor {
public abstract String getDateFormat(String dateType, String dateFormat, String column);
String getDateFormat(String dateType, String dateFormat, String column);
public abstract String getColumnMetaQueryTpl();
String functionNameCorrector(String sql);
public abstract String getDbMetaQueryTpl();
List<String> getDBs(ConnectInfo connectInfo) throws SQLException;
public abstract String getTableMetaQueryTpl();
List<String> getTables(ConnectInfo connectInfo, String schemaName) throws SQLException;
List<DBColumn> getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException;
public abstract String functionNameCorrector(String sql);
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java
View File

@@ -16,6 +16,7 @@ public class DbAdaptorFactory {
dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor());
dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor());
dbAdaptorMap.put(EngineType.POSTGRESQL.getName(), new PostgresqlAdaptor());
dbAdaptorMap.put(EngineType.OTHER.getName(), new DefaultDbAdaptor());
}
public static DbAdaptor getEngineAdaptor(String engineType) {

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DefaultDbAdaptor.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.headless.core.adaptor.db;
public class DefaultDbAdaptor extends BaseDbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
return column;
}
@Override
public String functionNameCorrector(String sql) {
return sql;
}
}

25
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
public class H2Adaptor extends DbAdaptor {
public class H2Adaptor extends BaseDbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
@@ -27,30 +27,9 @@ public class H2Adaptor extends DbAdaptor {
return column;
}
@Override
public String getColumnMetaQueryTpl() {
return "SELECT COLUMN_NAME AS name, "
+ " case DATA_TYPE"
+ " when '12' then 'varchar'"
+ " when '-5' then 'integer'"
+ " when '8' then 'double'"
+ " end AS dataType"
+ " FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ='%s' AND TABLE_NAME = '%s'";
}
@Override
public String getDbMetaQueryTpl() {
return "SELECT DISTINCT TABLE_SCHEMA as name FROM INFORMATION_SCHEMA.TABLES WHERE STORAGE_TYPE = 'MEMORY'";
}
@Override
public String getTableMetaQueryTpl() {
return "SELECT TABLE_NAME as name FROM INFORMATION_SCHEMA.TABLES "
+ "WHERE STORAGE_TYPE = 'MEMORY' AND TABLE_SCHEMA = '%s'";
}
@Override
public String functionNameCorrector(String sql) {
return sql;
}
}

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java
View File

@@ -4,8 +4,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;
public class MysqlAdaptor extends DbAdaptor {
public class MysqlAdaptor extends BaseDbAdaptor {
/**
* transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY)
@@ -32,26 +31,9 @@ public class MysqlAdaptor extends DbAdaptor {
return column;
}
@Override
public String getDbMetaQueryTpl() {
return "select distinct TABLE_SCHEMA as name from information_schema.tables "
+ "where TABLE_SCHEMA not in ('information_schema','mysql','performance_schema','sys');";
}
@Override
public String getTableMetaQueryTpl() {
return "select TABLE_NAME as name from information_schema.tables where TABLE_SCHEMA = '%s';";
}
@Override
public String functionNameCorrector(String sql) {
return sql;
}
@Override
public String getColumnMetaQueryTpl() {
return "SELECT COLUMN_NAME as name, DATA_TYPE as dataType, COLUMN_COMMENT as comment "
+ "FROM information_schema.columns WHERE table_schema ='%s' AND table_name = '%s'";
}
}

57
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java
View File

@@ -1,16 +1,24 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
public class PostgresqlAdaptor extends DbAdaptor {
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.UnaryOperator;
public class PostgresqlAdaptor extends BaseDbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
@@ -34,17 +42,6 @@ public class PostgresqlAdaptor extends DbAdaptor {
return column;
}
@Override
public String getDbMetaQueryTpl() {
return " SELECT datname as name FROM pg_database WHERE datistemplate = false ;";
}
@Override
public String getTableMetaQueryTpl() {
return " SELECT table_name as name FROM information_schema.tables WHERE "
+ "table_schema = 'public' AND table_catalog = '%s' ; ";
}
@Override
public String functionNameCorrector(String sql) {
Map<String, String> functionMap = new HashMap<>();
@@ -79,10 +76,28 @@ public class PostgresqlAdaptor extends DbAdaptor {
return SqlReplaceHelper.replaceFunction(sql, functionMap, functionCall);
}
@Override
public String getColumnMetaQueryTpl() {
return " SELECT column_name as name, data_type as dataType FROM information_schema.columns "
+ "WHERE table_schema = 'public' AND table_catalog = '%s' AND table_name = '%s' ; ";
public List<String> getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException {
List<String> tables = Lists.newArrayList();
DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo);
ResultSet tableSet = metaData.getTables(null, null, null, new String[]{"TABLE"});
while (tableSet.next()) {
String tableName = tableSet.getString("TABLE_NAME");
tables.add(tableName);
}
return tables;
}
public List<DBColumn> getColumns(ConnectInfo connectInfo, String schemaName, String tableName) throws SQLException {
List<DBColumn> dbColumns = Lists.newArrayList();
DatabaseMetaData metaData = getDatabaseMetaData(connectInfo);
ResultSet columns = metaData.getColumns(null, null, tableName, null);
while (columns.next()) {
String columnName = columns.getString("COLUMN_NAME");
String dataType = columns.getString("TYPE_NAME");
String remarks = columns.getString("REMARKS");
dbColumns.add(new DBColumn(columnName, dataType, remarks));
}
return dbColumns;
}
}