From 33268bf3d9481445a2bd87b6f7a677bb1bba4870 Mon Sep 17 00:00:00 2001 From: Hyman_bz Date: Wed, 19 Feb 2025 18:00:22 +0800 Subject: [PATCH] feat: add support starrocks and multiple catalog (#2066) --- .../common/pojo/enums/EngineType.java | 3 +- .../core/adaptor/db/BaseDbAdaptor.java | 15 +++- .../headless/core/adaptor/db/DbAdaptor.java | 4 +- .../core/adaptor/db/DbAdaptorFactory.java | 1 + .../core/adaptor/db/StarrocksAdaptor.java | 42 ++++++++++ .../server/pojo/DbParameterFactory.java | 1 + .../pojo/StarrocksParametersBuilder.java | 16 ++++ .../server/rest/DatabaseController.java | 10 ++- .../server/service/DatabaseService.java | 4 +- .../service/impl/DatabaseServiceImpl.java | 11 ++- .../Datasource/components/ModelBasicForm.tsx | 80 +++++++++++++++++-- .../src/pages/SemanticModel/service.ts | 12 ++- 12 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/StarrocksParametersBuilder.java diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java index 09de94ca5..6e60a16ac 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java @@ -9,7 +9,8 @@ public enum EngineType { POSTGRESQL(6, "POSTGRESQL"), OTHER(7, "OTHER"), DUCKDB(8, "DUCKDB"), - HANADB(9, "HANADB"); + HANADB(9, "HANADB"), + STARROCKS(10, "STARROCKS"),; private Integer code; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java index a61cc7d76..ac9676e67 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/BaseDbAdaptor.java @@ -17,7 +17,20 @@ import java.util.List; @Slf4j public abstract class BaseDbAdaptor implements DbAdaptor { - public List getDBs(ConnectInfo connectionInfo) throws SQLException { + @Override + public List getCatalogs(ConnectInfo connectInfo) throws SQLException { + // Apart from supporting multiple catalog types of data sources, other types will return an + // empty set by default. + return List.of(); + } + + public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { + // Except for special types implemented separately, the generic logic catalog does not take + // effect. + return getDBs(connectionInfo); + } + + protected List getDBs(ConnectInfo connectionInfo) throws SQLException { List dbs = Lists.newArrayList(); DatabaseMetaData metaData = getDatabaseMetaData(connectionInfo); try { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java index 94862f633..56ad9a777 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java @@ -14,7 +14,9 @@ public interface DbAdaptor { String rewriteSql(String sql); - List getDBs(ConnectInfo connectInfo) throws SQLException; + List getCatalogs(ConnectInfo connectInfo) throws SQLException; + + List getDBs(ConnectInfo connectInfo, String catalog) throws SQLException; List getTables(ConnectInfo connectInfo, String schemaName) throws SQLException; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java index 8e3eb08e7..a5694659e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java @@ -18,6 +18,7 @@ public class DbAdaptorFactory { dbAdaptorMap.put(EngineType.OTHER.getName(), new DefaultDbAdaptor()); dbAdaptorMap.put(EngineType.DUCKDB.getName(), new DuckdbAdaptor()); dbAdaptorMap.put(EngineType.HANADB.getName(), new HanadbAdaptor()); + dbAdaptorMap.put(EngineType.STARROCKS.getName(), new StarrocksAdaptor()); } public static DbAdaptor getEngineAdaptor(String engineType) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java new file mode 100644 index 000000000..55526893e --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/StarrocksAdaptor.java @@ -0,0 +1,42 @@ +package com.tencent.supersonic.headless.core.adaptor.db; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.core.pojo.ConnectInfo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.Assert; + +import java.sql.*; +import java.util.List; + +@Slf4j +public class StarrocksAdaptor extends MysqlAdaptor { + + @Override + public List getCatalogs(ConnectInfo connectInfo) throws SQLException { + List catalogs = Lists.newArrayList(); + try (Connection con = DriverManager.getConnection(connectInfo.getUrl(), + connectInfo.getUserName(), connectInfo.getPassword()); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery("SHOW CATALOGS")) { + while (rs.next()) { + catalogs.add(rs.getString(1)); + } + } + return catalogs; + } + + @Override + public List getDBs(ConnectInfo connectionInfo, String catalog) throws SQLException { + Assert.hasText(catalog, "StarRocks type catalog can not be null or empty"); + List dbs = Lists.newArrayList(); + try (Connection con = DriverManager.getConnection(connectionInfo.getUrl(), + connectionInfo.getUserName(), connectionInfo.getPassword()); + Statement st = con.createStatement(); + ResultSet rs = st.executeQuery("SHOW DATABASES IN " + catalog)) { + while (rs.next()) { + dbs.add(rs.getString(1)); + } + } + return dbs; + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java index 072023a42..66c8d7995 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java @@ -16,6 +16,7 @@ public class DbParameterFactory { parametersBuilder.put(EngineType.MYSQL.getName(), new MysqlParametersBuilder()); parametersBuilder.put(EngineType.POSTGRESQL.getName(), new PostgresqlParametersBuilder()); parametersBuilder.put(EngineType.HANADB.getName(), new HanadbParametersBuilder()); + parametersBuilder.put(EngineType.STARROCKS.getName(), new StarrocksParametersBuilder()); parametersBuilder.put(EngineType.OTHER.getName(), new OtherParametersBuilder()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/StarrocksParametersBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/StarrocksParametersBuilder.java new file mode 100644 index 000000000..4fe07ab2d --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/StarrocksParametersBuilder.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class StarrocksParametersBuilder extends DefaultParametersBuilder { + + @Override + public List build() { + return super.build(); + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 457845aad..b582e25e5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -76,9 +76,15 @@ public class DatabaseController { return databaseService.executeSql(sqlExecuteReq, user); } + @RequestMapping("/getCatalogs") + public List getCatalogs(@RequestParam("id") Long databaseId) throws SQLException { + return databaseService.getCatalogs(databaseId); + } + @RequestMapping("/getDbNames") - public List getDbNames(@RequestParam("id") Long databaseId) throws SQLException { - return databaseService.getDbNames(databaseId); + public List getDbNames(@RequestParam("id") Long databaseId, + @RequestParam(value = "catalog", required = false) String catalog) throws SQLException { + return databaseService.getDbNames(databaseId, catalog); } @RequestMapping("/getTables") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index 3b11edac4..5a57878a0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -36,7 +36,9 @@ public interface DatabaseService { void deleteDatabase(Long databaseId); - List getDbNames(Long id) throws SQLException; + List getCatalogs(Long id) throws SQLException; + + List getDbNames(Long id, String catalog) throws SQLException; List getTables(Long id, String db) throws SQLException; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 37a0c77da..93cb96975 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -200,10 +200,17 @@ public class DatabaseServiceImpl extends ServiceImpl getDbNames(Long id) throws SQLException { + public List getCatalogs(Long id) throws SQLException { DatabaseResp databaseResp = getDatabase(id); DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - return dbAdaptor.getDBs(DatabaseConverter.getConnectInfo(databaseResp)); + return dbAdaptor.getCatalogs(DatabaseConverter.getConnectInfo(databaseResp)); + } + + @Override + public List getDbNames(Long id, String catalog) throws SQLException { + DatabaseResp databaseResp = getDatabase(id); + DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); + return dbAdaptor.getDBs(DatabaseConverter.getConnectInfo(databaseResp), catalog); } @Override diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx index 0c9f248dc..1aa33143d 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/Datasource/components/ModelBasicForm.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from 'react'; import { Form, Input, Spin, Select, message } from 'antd'; import type { FormInstance } from 'antd/lib/form'; -import { getDbNames, getTables, getDimensionList } from '../../service'; +import {getDbNames, getTables, getDimensionList, getCatalogs} from '../../service'; import { ISemantic } from '../../data'; import FormItemTitle from '@/components/FormHelper/FormItemTitle'; @@ -20,13 +20,16 @@ const ModelBasicForm: React.FC = ({ isEdit, modelItem, databaseConfigList, + form, mode = 'normal', }) => { const [currentDbLinkConfigId, setCurrentDbLinkConfigId] = useState(); + const [catalogList, setCatalogList] = useState([]); const [dbNameList, setDbNameList] = useState([]); const [tableNameList, setTableNameList] = useState([]); const [loading, setLoading] = useState(false); const [dimensionOptions, setDimensionOptions] = useState<{ label: string; value: number }[]>([]); + const [catalogSelectOpen, setCatalogSelectOpen] = useState(false); useEffect(() => { if (modelItem?.id) { @@ -50,9 +53,49 @@ const ModelBasicForm: React.FC = ({ } }; - const queryDbNameList = async (databaseId: number) => { + const onDatabaseSelect = (databaseId: number, type: string) => { setLoading(true); - const { code, data, msg } = await getDbNames(databaseId); + if (type === 'STARROCKS') { + queryCatalogList(databaseId) + setCatalogSelectOpen(true); + setDbNameList([]); + } else { + queryDbNameList(databaseId, ""); + setCatalogSelectOpen(false); + setCatalogList([]); + } + form.setFieldsValue({ + catalog: undefined, + dbName: undefined, + tableName: undefined, + }) + }; + + const queryCatalogList = async (databaseId: number) => { + setLoading(true); + const { code, data, msg } = await getCatalogs(databaseId); + setLoading(false) + if (code === 200) { + const list = data || []; + setCatalogList(list); + } else { + message.error(msg); + } + } + + const onCatalogSelect = (catalog: string) => { + if (currentDbLinkConfigId) { + queryDbNameList(currentDbLinkConfigId, catalog); + } + form.setFieldsValue({ + dbName: undefined, + tableName: undefined, + }) + } + + const queryDbNameList = async (databaseId: number, catalog: string) => { + setLoading(true); + const { code, data, msg } = await getDbNames(databaseId, catalog); setLoading(false); if (code === 200) { const list = data || []; @@ -61,6 +104,7 @@ const ModelBasicForm: React.FC = ({ message.error(msg); } }; + const queryTableNameList = async (databaseName: string) => { if (!currentDbLinkConfigId) { return; @@ -89,18 +133,37 @@ const ModelBasicForm: React.FC = ({ showSearch placeholder="请选择数据库连接" disabled={isEdit} - onChange={(dbLinkConfigId: number) => { - queryDbNameList(dbLinkConfigId); + onSelect={(dbLinkConfigId: number, option) => { + onDatabaseSelect(dbLinkConfigId, option.type); setCurrentDbLinkConfigId(dbLinkConfigId); }} > {databaseConfigList.map((item) => ( - + {item.name} ))} + = ({ showSearch placeholder="请先选择一个数据库连接" disabled={isEdit} - onChange={(dbName: string) => { + onSelect={(dbName: string) => { queryTableNameList(dbName); + form.setFieldsValue({ + tableName: undefined, + }) }} > {dbNameList.map((item) => ( diff --git a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts index eb29cfa14..c650b4c8b 100644 --- a/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts +++ b/webapp/packages/supersonic-fe/src/pages/SemanticModel/service.ts @@ -379,11 +379,21 @@ export async function listColumnsBySql(data: { databaseId: number; sql: string } }); } -export function getDbNames(dbId: number): Promise { +export function getCatalogs(dbId: number): Promise { + return request(`${process.env.API_BASE_URL}database/getCatalogs`, { + method: 'GET', + params: { + id: dbId, + }, + }); +} + +export function getDbNames(dbId: number, catalog: string): Promise { return request(`${process.env.API_BASE_URL}database/getDbNames`, { method: 'GET', params: { id: dbId, + catalog: catalog, }, }); }