From 5e3bafb95344459af8baada1e86dbcffa823a2b0 Mon Sep 17 00:00:00 2001 From: zyclove Date: Wed, 26 Feb 2025 17:33:14 +0800 Subject: [PATCH] feat:Support kyuubi presto trino (#2109) --- .../common/pojo/enums/EngineType.java | 5 +- .../headless/api/pojo/DbSchema.java | 2 + .../headless/api/pojo/enums/DataType.java | 7 +- .../api/pojo/request/ModelBuildReq.java | 2 + .../chat/mapper/EmbeddingMatchStrategy.java | 6 +- .../headless/chat/mapper/MapFilter.java | 5 +- .../headless/chat/mapper/MapperConfig.java | 4 +- headless/core/pom.xml | 12 ++ .../core/adaptor/db/BaseDbAdaptor.java | 116 ++++++++++++------ .../headless/core/adaptor/db/DbAdaptor.java | 5 +- .../core/adaptor/db/DbAdaptorFactory.java | 3 + .../core/adaptor/db/DuckdbAdaptor.java | 2 +- .../headless/core/adaptor/db/H2Adaptor.java | 2 +- .../core/adaptor/db/KyuubiAdaptor.java | 84 +++++++++++++ .../core/adaptor/db/PostgresqlAdaptor.java | 2 +- .../core/adaptor/db/PrestoAdaptor.java | 88 +++++++++++++ .../core/adaptor/db/StarrocksAdaptor.java | 83 ++++++++++--- .../core/adaptor/db/TrinoAdaptor.java | 8 ++ .../core/utils/JdbcDataSourceUtils.java | 21 +++- .../server/pojo/DbParameterFactory.java | 3 + .../server/pojo/DefaultParametersBuilder.java | 1 + .../server/pojo/KyuubiParametersBuilder.java | 16 +++ .../server/pojo/PrestoParametersBuilder.java | 16 +++ .../server/pojo/TrinoParametersBuilder.java | 16 +++ .../server/rest/DatabaseController.java | 9 +- .../server/service/DatabaseService.java | 4 +- .../service/impl/DatabaseServiceImpl.java | 14 +-- pom.xml | 18 +++ .../Datasource/components/ModelBasicForm.tsx | 10 +- .../Datasource/components/ModelCreateForm.tsx | 32 +++-- .../src/pages/SemanticModel/service.ts | 6 +- 31 files changed, 501 insertions(+), 101 deletions(-) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/KyuubiAdaptor.java create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/TrinoAdaptor.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/KyuubiParametersBuilder.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PrestoParametersBuilder.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TrinoParametersBuilder.java diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java index 6e60a16ac..ac91bfac9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java @@ -10,7 +10,10 @@ public enum EngineType { OTHER(7, "OTHER"), DUCKDB(8, "DUCKDB"), HANADB(9, "HANADB"), - STARROCKS(10, "STARROCKS"),; + STARROCKS(10, "STARROCKS"), + KYUUBI(11, "KYUUBI"), + PRESTO(12, "PRESTO"), + TRINO(13, "TRINO"),; private Integer code; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DbSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DbSchema.java index 98dad8c3b..40afe4d99 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DbSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DbSchema.java @@ -11,6 +11,8 @@ import java.util.List; @NoArgsConstructor public class DbSchema { + private String catalog; + private String db; private String table; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java index 2670eec81..e7a9a5b81 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java @@ -8,7 +8,9 @@ import java.util.Set; public enum DataType { MYSQL("mysql", "mysql", "com.mysql.cj.jdbc.Driver", "`", "`", "'", "'"), - HIVE2("hive2", "hive", "org.apache.hive.jdbc.HiveDriver", "`", "`", "`", "`"), + HIVE2("hive2", "hive", "org.apache.kyuubi.jdbc.KyuubiHiveDriver", "`", "`", "`", "`"), + + KYUUBI("kyuubi", "kyuubi", "org.apache.kyuubi.jdbc.KyuubiHiveDriver", "`", "`", "`", "`"), ORACLE("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), @@ -27,6 +29,8 @@ public enum DataType { PRESTO("presto", "presto", "com.facebook.presto.jdbc.PrestoDriver", "\"", "\"", "\"", "\""), + TRINO("trino", "trino", "io.trino.jdbc.TrinoDriver", "\"", "\"", "\"", "\""), + MOONBOX("moonbox", "moonbox", "moonbox.jdbc.MbDriver", "`", "`", "`", "`"), CASSANDRA("cassandra", "cassandra", "com.github.adejanovski.cassandra.jdbc.CassandraDriver", "", @@ -46,6 +50,7 @@ public enum DataType { TDENGINE("TAOS", "TAOS", "com.taosdata.jdbc.TSDBDriver", "'", "'", "\"", "\""), POSTGRESQL("postgresql", "postgresql", "org.postgresql.Driver", "'", "'", "\"", "\""), + DUCKDB("duckdb", "duckdb", "org.duckdb.DuckDBDriver", "'", "'", "\"", "\""); private String feature; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java index 8dc9ffa0a..d0d0b48cb 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java @@ -19,6 +19,8 @@ public class ModelBuildReq { private String sql; + private String catalog; + private String db; private List tables; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java index 618ed2d1b..790b3bdd7 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java @@ -72,7 +72,8 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy // 1. Base detection List baseResults = super.detect(chatQueryContext, terms, detectDataSetIds); - boolean useLLM = Boolean.parseBoolean(mapperConfig.getParameterValue(EMBEDDING_MAPPER_USE_LLM)); + boolean useLLM = + Boolean.parseBoolean(mapperConfig.getParameterValue(EMBEDDING_MAPPER_USE_LLM)); // 2. LLM enhanced detection if (useLLM) { @@ -115,7 +116,8 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy * Extract valid word segments by filtering out unwanted word natures */ private Set extractValidSegments(String text) { - List natureList = Arrays.asList(StringUtils.split(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ALLOWED_SEGMENT_NATURE ), ",")); + List natureList = Arrays.asList(StringUtils.split( + mapperConfig.getParameterValue(EMBEDDING_MAPPER_ALLOWED_SEGMENT_NATURE), ",")); return HanlpHelper.getSegment().seg(text).stream() .filter(t -> natureList.stream().noneMatch(nature -> t.nature.startsWith(nature))) .map(Term::getWord).collect(Collectors.toSet()); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java index b16a6f186..aad8d10c4 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java @@ -61,7 +61,8 @@ public class MapFilter { List value = entry.getValue(); if (!CollectionUtils.isEmpty(value)) { value.removeIf(schemaElementMatch -> StringUtils - .length(schemaElementMatch.getDetectWord()) <= 1 && !schemaElementMatch.isLlmMatched()); + .length(schemaElementMatch.getDetectWord()) <= 1 + && !schemaElementMatch.isLlmMatched()); } } } @@ -80,7 +81,7 @@ public class MapFilter { } public static void filterByQueryDataType(ChatQueryContext chatQueryContext, - Predicate needRemovePredicate) { + Predicate needRemovePredicate) { Map> dataSetElementMatches = chatQueryContext.getMapInfo().getDataSetElementMatches(); for (Map.Entry> entry : dataSetElementMatches.entrySet()) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java index cd4b6772c..8a9feda56 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java @@ -63,6 +63,6 @@ public class MapperConfig extends ParameterConfig { "embedding的结果再通过一次LLM来筛选，这时候忽略各个向量阀值", "bool", "Mapper相关配置"); public static final Parameter EMBEDDING_MAPPER_ALLOWED_SEGMENT_NATURE = - new Parameter("s2.mapper.embedding.allowed-segment-nature", "['v', 'd', 'a']", "使用LLM召回二次处理时对问题分词词性的控制", - "分词后允许的词性才会进行向量召回", "list", "Mapper相关配置"); + new Parameter("s2.mapper.embedding.allowed-segment-nature", "['v', 'd', 'a']", + "使用LLM召回二次处理时对问题分词词性的控制", "分词后允许的词性才会进行向量召回", "list", "Mapper相关配置"); } diff --git a/headless/core/pom.xml b/headless/core/pom.xml index 8f24644ff..d261b6e2f 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -121,6 +121,18 @@ DmJdbcDriver18 8.1.2.192 + + org.apache.kyuubi + kyuubi-hive-jdbc + + + com.facebook.presto + presto-jdbc + + + io.trino + trino-jdbc + diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java index ac9676e67..6c661cf2f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java @@ -5,23 +5,27 @@ import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.enums.FieldType; import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.ArrayList; import java.util.List; +import java.util.Properties; @Slf4j public abstract class BaseDbAdaptor implements DbAdaptor { @Override public List getCatalogs(ConnectInfo connectInfo) throws SQLException { - // Apart from supporting multiple catalog types of data sources, other types will return an - // empty set by default. - return List.of(); + List catalogs = Lists.newArrayList(); + try (Connection con = getConnection(connectInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery("SHOW CATALOGS")) { + while (rs.next()) { + catalogs.add(rs.getString(1)); + } + } + return catalogs; } public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { @@ -32,38 +36,49 @@ public abstract class BaseDbAdaptor implements DbAdaptor { protected List getDBs(ConnectInfo connectionInfo) throws SQLException { List dbs = Lists.newArrayList(); - DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); try { - ResultSet schemaSet = metaData.getSchemas(); - while (schemaSet.next()) { - String db = schemaSet.getString("TABLE_SCHEM"); - dbs.add(db); + try (ResultSet schemaSet = getDatabaseMetaData(connectionInfo).getSchemas()) { + while (schemaSet.next()) { + String db = schemaSet.getString("TABLE_SCHEM"); + dbs.add(db); + } } } catch (Exception e) { - log.info("get meta schemas failed, try to get catalogs"); + log.warn("get meta schemas failed", e); + log.warn("get meta schemas failed, try to get catalogs"); } try { - ResultSet catalogSet = metaData.getCatalogs(); - while (catalogSet.next()) { - String db = catalogSet.getString("TABLE_CAT"); - dbs.add(db); + try (ResultSet catalogSet = getDatabaseMetaData(connectionInfo).getCatalogs()) { + while (catalogSet.next()) { + String db = catalogSet.getString("TABLE_CAT"); + dbs.add(db); + } } } catch (Exception e) { - log.info("get meta catalogs failed, try to get schemas"); + log.warn("get meta catalogs failed", e); + log.warn("get meta catalogs failed, try to get schemas"); } return dbs; } - public List getTables(ConnectInfo connectionInfo, String schemaName) + @Override + public List getTables(ConnectInfo connectInfo, String catalog, String schemaName) + throws SQLException { + // Except for special types implemented separately, the generic logic catalog does not take + // effect. + return getTables(connectInfo, schemaName); + } + + protected List getTables(ConnectInfo connectionInfo, String schemaName) throws SQLException { List tablesAndViews = new ArrayList<>(); - DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); try { - ResultSet resultSet = getResultSet(schemaName, metaData); - while (resultSet.next()) { - String name = resultSet.getString("TABLE_NAME"); - tablesAndViews.add(name); + try(ResultSet resultSet = getResultSet(schemaName, getDatabaseMetaData(connectionInfo))) { + while (resultSet.next()) { + String name = resultSet.getString("TABLE_NAME"); + tablesAndViews.add(name); + } } } catch (SQLException e) { log.error("Failed to get tables and views", e); @@ -76,27 +91,34 @@ public abstract class BaseDbAdaptor implements DbAdaptor { return metaData.getTables(schemaName, schemaName, null, new String[] {"TABLE", "VIEW"}); } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + + + public List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) throws SQLException { - List dbColumns = Lists.newArrayList(); - DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); - ResultSet columns = metaData.getColumns(schemaName, schemaName, tableName, null); - while (columns.next()) { - String columnName = columns.getString("COLUMN_NAME"); - String dataType = columns.getString("TYPE_NAME"); - String remarks = columns.getString("REMARKS"); - FieldType fieldType = classifyColumnType(dataType); - dbColumns.add(new DBColumn(columnName, dataType, remarks, fieldType)); + List dbColumns = new ArrayList<>(); + // 确保连接会自动关闭 + try (ResultSet columns = getDatabaseMetaData(connectInfo).getColumns(catalog, schemaName, tableName, null)) { + while (columns.next()) { + String columnName = columns.getString("COLUMN_NAME"); + String dataType = columns.getString("TYPE_NAME"); + String remarks = columns.getString("REMARKS"); + FieldType fieldType = classifyColumnType(dataType); + dbColumns.add(new DBColumn(columnName, dataType, remarks, fieldType)); + } } return dbColumns; } protected DatabaseMetaData getDatabaseMetaData(ConnectInfo connectionInfo) throws SQLException { - Connection connection = DriverManager.getConnection(connectionInfo.getUrl(), - connectionInfo.getUserName(), connectionInfo.getPassword()); + Connection connection = getConnection(connectionInfo); return connection.getMetaData(); } + public Connection getConnection(ConnectInfo connectionInfo) throws SQLException { + final Properties properties = getProperties(connectionInfo); + return DriverManager.getConnection(connectionInfo.getUrl(), properties); + } + public FieldType classifyColumnType(String typeName) { switch (typeName.toUpperCase()) { case "INT": @@ -118,4 +140,24 @@ public abstract class BaseDbAdaptor implements DbAdaptor { } } + public Properties getProperties(ConnectInfo connectionInfo) { + final Properties properties = new Properties(); + String url = connectionInfo.getUrl().toLowerCase(); + + // 设置通用属性 + properties.setProperty("user", connectionInfo.getUserName()); + + // 针对 Presto 和 Trino ssl=false 的情况，不需要设置密码 + if (url.startsWith("jdbc:presto") || url.startsWith("jdbc:trino")) { + // 检查是否需要处理 SSL + if (!url.contains("ssl=false")) { + properties.setProperty("password", connectionInfo.getPassword()); + } + } else { + // 针对其他数据库类型 + properties.setProperty("password", connectionInfo.getPassword()); + } + + return properties; + } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index 56ad9a777..b90cadf1e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -18,9 +18,10 @@ public interface DbAdaptor { List getDBs(ConnectInfo connectInfo, String catalog) throws SQLException; - List getTables(ConnectInfo connectInfo, String schemaName) throws SQLException; + List getTables(ConnectInfo connectInfo, String catalog, String schemaName) + throws SQLException; - List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) throws SQLException; FieldType classifyColumnType(String typeName); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java index a5694659e..909c64fb1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java @@ -19,6 +19,9 @@ public class DbAdaptorFactory { dbAdaptorMap.put(EngineType.DUCKDB.getName(), new DuckdbAdaptor()); dbAdaptorMap.put(EngineType.HANADB.getName(), new HanadbAdaptor()); dbAdaptorMap.put(EngineType.STARROCKS.getName(), new StarrocksAdaptor()); + dbAdaptorMap.put(EngineType.KYUUBI.getName(), new KyuubiAdaptor()); + dbAdaptorMap.put(EngineType.PRESTO.getName(), new PrestoAdaptor()); + dbAdaptorMap.put(EngineType.TRINO.getName(), new TrinoAdaptor()); } public static DbAdaptor getEngineAdaptor(String engineType) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DuckdbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DuckdbAdaptor.java index d89193b57..b4359103a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DuckdbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DuckdbAdaptor.java @@ -19,7 +19,7 @@ public class DuckdbAdaptor extends DefaultDbAdaptor { return metaData.getTables(schemaName, null, null, new String[] {"TABLE", "VIEW"}); } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + public List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) throws SQLException { List dbColumns = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java index 5d7e4151d..917a63b28 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java @@ -46,7 +46,7 @@ public class H2Adaptor extends BaseDbAdaptor { return metaData.getTables(schemaName, null, null, new String[] {"TABLE", "VIEW"}); } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + public List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) throws SQLException { List dbColumns = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/KyuubiAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/KyuubiAdaptor.java new file mode 100644 index 000000000..ce7958717 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/KyuubiAdaptor.java @@ -0,0 +1,84 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.api.pojo.enums.FieldType; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class KyuubiAdaptor extends BaseDbAdaptor { + + /** transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY) */ + @Override + public String getDateFormat(String dateType, String dateFormat, String column) { + if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { + if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(%s, 'yyyy-MM')", column); + } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(date_sub(%s, (dayofweek(%s) - 2)), 'yyyy-MM-dd')", + column, column); + } else { + return String.format( + "date_format(to_date(cast(%s as string), 'yyyyMMdd'), 'yyyy-MM-dd')", + column); + } + } else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) { + if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(%s, 'yyyy-MM')", column); + } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(date_sub(%s, (dayofweek(%s) - 2)), 'yyyy-MM-dd')", + column, column); + } else { + return column; + } + } + return column; + } + + @Override + public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { + List dbs = Lists.newArrayList(); + final StringBuilder sql = new StringBuilder("SHOW DATABASES"); + if (StringUtils.isNotBlank(catalog)) { + sql.append(" IN ").append(catalog); + } + try (Connection con = getConnection(connectionInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery(sql.toString())) { + while (rs.next()) { + dbs.add(rs.getString(1)); + } + } + return dbs; + } + + @Override + public List getTables(ConnectInfo connectInfo, String catalog, String schemaName) throws SQLException { + List tablesAndViews = new ArrayList<>(); + + try { + try (ResultSet resultSet = getDatabaseMetaData(connectInfo).getTables(catalog, schemaName, null, new String[] {"TABLE", "VIEW"})) { + while (resultSet.next()) { + String name = resultSet.getString("TABLE_NAME"); + tablesAndViews.add(name); + } + } + } catch (SQLException e) { + log.error("Failed to get tables and views", e); + } + return tablesAndViews; + } + + @Override + public String rewriteSql(String sql) { + return sql; + } +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java index f04fb0e5f..1b9658ee6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java @@ -99,7 +99,7 @@ public class PostgresqlAdaptor extends BaseDbAdaptor { return tablesAndViews; } - public List getColumns(ConnectInfo connectInfo, String schemaName, String tableName) + public List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) throws SQLException { List dbColumns = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectInfo); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java new file mode 100644 index 000000000..c80483ee7 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java @@ -0,0 +1,88 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.api.pojo.enums.FieldType; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import org.apache.commons.lang3.StringUtils; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +public class PrestoAdaptor extends BaseDbAdaptor { + + /** transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY) */ + @Override + public String getDateFormat(String dateType, String dateFormat, String column) { + if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) { + if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(%s, '%%Y-%%m')", column); + } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { + return String.format( + "date_format(date_add('day', - (day_of_week(%s) - 2), %s), '%%Y-%%m-%%d')", + column, column); + } else { + return String.format("date_format(date_parse(%s, '%%Y%%m%%d'), '%%Y-%%m-%%d')", + column); + } + } else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) { + if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) { + return String.format("date_format(%s, '%%Y-%%m')", column); + } else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) { + return String.format( + "date_format(date_add('day', - (day_of_week(%s) - 2), %s), '%%Y-%%m-%%d')", + column, column); + } else { + return column; + } + } + return column; + } + + @Override + public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { + List dbs = Lists.newArrayList(); + final StringBuilder sql = new StringBuilder("SHOW SCHEMAS"); + if (StringUtils.isNotBlank(catalog)) { + sql.append(" IN ").append(catalog); + } + try (Connection con = getConnection(connectionInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery(sql.toString())) { + while (rs.next()) { + dbs.add(rs.getString(1)); + } + } + return dbs; + } + + @Override + public List getTables(ConnectInfo connectInfo, String catalog, String schemaName) + throws SQLException { + List tablesAndViews = new ArrayList<>(); + final StringBuilder sql = new StringBuilder("SHOW TABLES"); + if (StringUtils.isNotBlank(catalog)) { + sql.append(" IN ").append(catalog).append(".").append(schemaName); + }else { + sql.append(" IN ").append(schemaName); + } + + try (Connection con = getConnection(connectInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery(sql.toString())) { + while (rs.next()) { + tablesAndViews.add(rs.getString(1)); + } + } + return tablesAndViews; + } + + @Override + public String rewriteSql(String sql) { + return sql; + } +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java index 55526893e..75824cf42 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java @@ -1,42 +1,87 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.DBColumn; +import com.tencent.supersonic.headless.api.pojo.enums.FieldType; import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; import java.sql.*; +import java.util.ArrayList; import java.util.List; +import java.util.Properties; @Slf4j public class StarrocksAdaptor extends MysqlAdaptor { - @Override - public List getCatalogs(ConnectInfo connectInfo) throws SQLException { - List catalogs = Lists.newArrayList(); - try (Connection con = DriverManager.getConnection(connectInfo.getUrl(), - connectInfo.getUserName(), connectInfo.getPassword()); - Statement st = con.createStatement(); - ResultSet rs = st.executeQuery("SHOW CATALOGS")) { - while (rs.next()) { - catalogs.add(rs.getString(1)); - } - } - return catalogs; - } - @Override public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { - Assert.hasText(catalog, "StarRocks type catalog can not be null or empty"); List dbs = Lists.newArrayList(); - try (Connection con = DriverManager.getConnection(connectionInfo.getUrl(), - connectionInfo.getUserName(), connectionInfo.getPassword()); - Statement st = con.createStatement(); - ResultSet rs = st.executeQuery("SHOW DATABASES IN " + catalog)) { + final StringBuilder sql = new StringBuilder("SHOW DATABASES"); + if (StringUtils.isNotBlank(catalog)) { + sql.append(" IN ").append(catalog); + } + try (Connection con = getConnection(connectionInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery(sql.toString())) { while (rs.next()) { dbs.add(rs.getString(1)); } } return dbs; } + + @Override + public List getTables(ConnectInfo connectInfo, String catalog, String schemaName) + throws SQLException { + List tablesAndViews = new ArrayList<>(); + final StringBuilder sql = new StringBuilder("SHOW TABLES"); + if (StringUtils.isNotBlank(catalog)) { + sql.append(" IN ").append(catalog).append(".").append(schemaName); + }else { + sql.append(" IN ").append(schemaName); + } + + try (Connection con = getConnection(connectInfo); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery(sql.toString())) { + while (rs.next()) { + tablesAndViews.add(rs.getString(1)); + } + } + return tablesAndViews; + } + + @Override + public List getColumns(ConnectInfo connectInfo, String catalog, String schemaName, String tableName) + throws SQLException { + List dbColumns = new ArrayList<>(); + + try (Connection con = getConnection(connectInfo); + Statement st = con.createStatement()) { + + // 切换到指定的 catalog（或 database/schema），这在某些 SQL 方言中很重要 + if (StringUtils.isNotBlank(catalog)) { + st.execute("SET CATALOG " + catalog); + } + + // 获取 DatabaseMetaData; 需要注意调用此方法的位置（在 USE 之后） + DatabaseMetaData metaData = con.getMetaData(); + + // 获取特定表的列信息 + try (ResultSet columns = metaData.getColumns(schemaName, schemaName, tableName, null)) { + while (columns.next()) { + String columnName = columns.getString("COLUMN_NAME"); + String dataType = columns.getString("TYPE_NAME"); + String remarks = columns.getString("REMARKS"); + FieldType fieldType = classifyColumnType(dataType); + dbColumns.add(new DBColumn(columnName, dataType, remarks, fieldType)); + } + } + } + + return dbColumns; + } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/TrinoAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/TrinoAdaptor.java new file mode 100644 index 000000000..d42c14cc8 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/TrinoAdaptor.java @@ -0,0 +1,8 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; + +public class TrinoAdaptor extends PrestoAdaptor { + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java index 8dee1e82a..4859c20ab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java @@ -40,11 +40,22 @@ public class JdbcDataSourceUtils { log.error(e.toString(), e); return false; } - try (Connection con = DriverManager.getConnection(database.getUrl(), database.getUsername(), - database.passwordDecrypt())) { - return con != null; - } catch (SQLException e) { - log.error(e.toString(), e); + // presto/trino ssl=false connection need password + if (database.getUrl().startsWith("jdbc:presto") || database.getUrl().startsWith("jdbc:trino")) { + if (database.getUrl().toLowerCase().contains("ssl=false")) { + try (Connection con = DriverManager.getConnection(database.getUrl(), database.getUsername(), null)) { + return con != null; + } catch (SQLException e) { + log.error(e.toString(), e); + } + } + }else { + try (Connection con = DriverManager.getConnection(database.getUrl(), database.getUsername(), + database.passwordDecrypt())) { + return con != null; + } catch (SQLException e) { + log.error(e.toString(), e); + } } return false; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java index 66c8d7995..527ccbce7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java @@ -17,6 +17,9 @@ public class DbParameterFactory { parametersBuilder.put(EngineType.POSTGRESQL.getName(), new PostgresqlParametersBuilder()); parametersBuilder.put(EngineType.HANADB.getName(), new HanadbParametersBuilder()); parametersBuilder.put(EngineType.STARROCKS.getName(), new StarrocksParametersBuilder()); + parametersBuilder.put(EngineType.KYUUBI.getName(), new KyuubiParametersBuilder()); + parametersBuilder.put(EngineType.PRESTO.getName(), new PrestoParametersBuilder()); + parametersBuilder.put(EngineType.TRINO.getName(), new TrinoParametersBuilder()); parametersBuilder.put(EngineType.OTHER.getName(), new OtherParametersBuilder()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DefaultParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DefaultParametersBuilder.java index 8ac5ace28..b00fb0e42 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DefaultParametersBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DefaultParametersBuilder.java @@ -29,6 +29,7 @@ public class DefaultParametersBuilder implements DbParametersBuilder { password.setComment("密码"); password.setName("password"); password.setPlaceholder("请输入密码"); + password.setRequire(false); databaseParameters.add(password); return databaseParameters; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/KyuubiParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/KyuubiParametersBuilder.java new file mode 100644 index 000000000..d456963ff --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/KyuubiParametersBuilder.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class KyuubiParametersBuilder extends DefaultParametersBuilder { + + @Override + public List build() { + return super.build(); + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PrestoParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PrestoParametersBuilder.java new file mode 100644 index 000000000..72122604b --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PrestoParametersBuilder.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class PrestoParametersBuilder extends DefaultParametersBuilder { + + @Override + public List build() { + return super.build(); + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TrinoParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TrinoParametersBuilder.java new file mode 100644 index 000000000..a7a89b11e --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TrinoParametersBuilder.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class TrinoParametersBuilder extends DefaultParametersBuilder { + + @Override + public List build() { + return super.build(); + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index b582e25e5..834eff5bc 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -89,15 +89,18 @@ public class DatabaseController { @RequestMapping("/getTables") public List getTables(@RequestParam("databaseId") Long databaseId, + @RequestParam(value = "catalog", required = false) String catalog, @RequestParam("db") String db) throws SQLException { - return databaseService.getTables(databaseId, db); + return databaseService.getTables(databaseId, catalog, db); } @RequestMapping("/getColumnsByName") public List getColumnsByName(@RequestParam("databaseId") Long databaseId, - @RequestParam("db") String db, @RequestParam("table") String table) + @RequestParam(name="catalog", required = false) String catalog, + @RequestParam("db") String db, + @RequestParam("table") String table) throws SQLException { - return databaseService.getColumns(databaseId, db, table); + return databaseService.getColumns(databaseId, catalog, db, table); } @PostMapping("/listColumnsBySql") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index 5a57878a0..b5ef99dd9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -40,11 +40,11 @@ public interface DatabaseService { List getDbNames(Long id, String catalog) throws SQLException; - List getTables(Long id, String db) throws SQLException; + List getTables(Long id, String catalog, String db) throws SQLException; Map> getDbColumns(ModelBuildReq modelBuildReq) throws SQLException; - List getColumns(Long id, String db, String table) throws SQLException; + List getColumns(Long id, String catalog, String db, String table) throws SQLException; List getColumns(Long id, String sql) throws SQLException; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 93cb96975..fe73b8bb7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -214,10 +214,10 @@ public class DatabaseServiceImpl extends ServiceImpl getTables(Long id, String db) throws SQLException { + public List getTables(Long id, String catalog, String db) throws SQLException { DatabaseResp databaseResp = getDatabase(id); DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - return dbAdaptor.getTables(DatabaseConverter.getConnectInfo(databaseResp), db); + return dbAdaptor.getTables(DatabaseConverter.getConnectInfo(databaseResp), catalog, db); } @Override @@ -234,7 +234,7 @@ public class DatabaseServiceImpl extends ServiceImpl columns = - getColumns(modelBuildReq.getDatabaseId(), modelBuildReq.getDb(), table); + getColumns(modelBuildReq.getDatabaseId(), modelBuildReq.getCatalog(), modelBuildReq.getDb(), table); dbColumnMap.put(table, columns); } } @@ -242,15 +242,15 @@ public class DatabaseServiceImpl extends ServiceImpl getColumns(Long id, String db, String table) throws SQLException { + public List getColumns(Long id, String catalog, String db, String table) throws SQLException { DatabaseResp databaseResp = getDatabase(id); - return getColumns(databaseResp, db, table); + return getColumns(databaseResp, catalog, db, table); } - public List getColumns(DatabaseResp databaseResp, String db, String table) + public List getColumns(DatabaseResp databaseResp, String catalog, String db, String table) throws SQLException { DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - return engineAdaptor.getColumns(DatabaseConverter.getConnectInfo(databaseResp), db, table); + return engineAdaptor.getColumns(DatabaseConverter.getConnectInfo(databaseResp), catalog, db, table); } @Override diff --git a/pom.xml b/pom.xml index 12f6c5818..e3947fdc9 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,9 @@ 0.12.3 1.2.24 5.1.46 + 1.10.1 + 0.291 + 471 3.5.7 5.4.1 @@ -208,6 +211,21 @@ mysql-connector-java ${mysql.connector.java.version} + + org.apache.kyuubi + kyuubi-hive-jdbc + ${kyuubi.version} + + + com.facebook.presto + presto-jdbc + ${presto.version} + + + io.trino + trino-jdbc + ${trino.version} + org.mockito mockito-inline diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx index 81c557a4c..e3c9724aa 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx @@ -24,6 +24,7 @@ const ModelBasicForm: React.FC = ({ mode = 'normal', }) => { const [currentDbLinkConfigId, setCurrentDbLinkConfigId] = useState(); + const [currentCatalog, setCurrentCatalog] = useState(""); const [catalogList, setCatalogList] = useState([]); const [dbNameList, setDbNameList] = useState([]); const [tableNameList, setTableNameList] = useState([]); @@ -55,8 +56,8 @@ const ModelBasicForm: React.FC = ({ const onDatabaseSelect = (databaseId: number, type: string) => { setLoading(true); - if (type === 'STARROCKS') { - queryCatalogList(databaseId) + if (type === 'STARROCKS' || type === 'KYUUBI' || type === 'PRESTO' || type === 'TRINO') { + queryCatalogList(databaseId); setCatalogSelectOpen(true); setDbNameList([]); } else { @@ -88,9 +89,11 @@ const ModelBasicForm: React.FC = ({ queryDbNameList(currentDbLinkConfigId, catalog); } form.setFieldsValue({ + catalog: catalog, dbName: undefined, tableName: undefined, }) + setCurrentCatalog(catalog); } const queryDbNameList = async (databaseId: number, catalog: string) => { @@ -110,7 +113,7 @@ const ModelBasicForm: React.FC = ({ return; } setLoading(true); - const { code, data, msg } = await getTables(currentDbLinkConfigId, databaseName); + const { code, data, msg } = await getTables(currentDbLinkConfigId, currentCatalog, databaseName); setLoading(false); if (code === 200) { const list = data || []; @@ -136,6 +139,7 @@ const ModelBasicForm: React.FC = ({ onSelect={(dbLinkConfigId: number, option) => { onDatabaseSelect(dbLinkConfigId, option.type); setCurrentDbLinkConfigId(dbLinkConfigId); + setCurrentCatalog(""); }} > {databaseConfigList.map((item) => ( diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx index a0f33848c..3c6ec5fc1 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelCreateForm.tsx @@ -351,12 +351,24 @@ const ModelCreateForm: React.FC = ({ let columns = fieldColumns || []; if (queryType === 'table_query') { const tableQueryString = tableQuery || ''; - const [dbName, tableName] = tableQueryString.split('.'); - columns = await queryTableColumnList(modelItem.databaseId, dbName, tableName); - tableQueryInitValue = { - dbName, - tableName, - }; + if (tableQueryString.split('.').length === 3) { + const [catalog, dbName, tableName] = tableQueryString.split('.'); + columns = await queryTableColumnList(modelItem.databaseId, catalog, dbName, tableName); + tableQueryInitValue = { + catalog, + dbName, + tableName, + }; + } + if (tableQueryString.split('.').length === 2) { + const [dbName, tableName] = tableQueryString.split('.'); + columns = await queryTableColumnList(modelItem.databaseId, '', dbName, tableName); + tableQueryInitValue = { + catalog: '', + dbName, + tableName, + }; + } } formatterInitData(columns, tableQueryInitValue); }; @@ -426,8 +438,8 @@ const ModelCreateForm: React.FC = ({ setFields(result); }; - const queryTableColumnList = async (databaseId: number, dbName: string, tableName: string) => { - const { code, data, msg } = await getColumns(databaseId, dbName, tableName); + const queryTableColumnList = async (databaseId: number, catalog: string, dbName: string, tableName: string) => { + const { code, data, msg } = await getColumns(databaseId, catalog, dbName, tableName); if (code === 200) { const list = data || []; const columns = list.map((item: any, index: number) => { @@ -563,10 +575,10 @@ const ModelCreateForm: React.FC = ({ }} onValuesChange={(value, values) => { const { tableName } = value; - const { dbName, databaseId } = values; + const { catalog, dbName, databaseId } = values; setFormDatabaseId(databaseId); if (tableName) { - queryTableColumnList(databaseId, dbName, tableName); + queryTableColumnList(databaseId, catalog, dbName, tableName); } }} className={styles.form} diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts index c650b4c8b..e2cee2db9 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts @@ -398,21 +398,23 @@ export function getDbNames(dbId: number, catalog: string): Promise { }); } -export function getTables(databaseId: number, dbName: string): Promise { +export function getTables(databaseId: number, catalog: string, dbName: string): Promise { return request(`${process.env.API_BASE_URL}database/getTables`, { method: 'GET', params: { databaseId, + catalog: catalog, db: dbName, }, }); } -export function getColumns(databaseId: number, dbName: string, tableName: string): Promise { +export function getColumns(databaseId: number, catalog: string, dbName: string, tableName: string): Promise { return request(`${process.env.API_BASE_URL}database/getColumnsByName`, { method: 'GET', params: { databaseId, + catalog: catalog, db: dbName, table: tableName, },