topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) support more db type (#1511)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-08-06 20:24:49 +08:00
committed by GitHub
parent 45fb83356f
commit c8fe6d2d04
17 changed files with 296 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java
View File

@@ -15,6 +15,7 @@ public class DbParameterFactory {
parametersBuilder.put(EngineType.CLICKHOUSE.getName(), new ClickHouseParametersBuilder());
parametersBuilder.put(EngineType.MYSQL.getName(), new MysqlParametersBuilder());
parametersBuilder.put(EngineType.POSTGRESQL.getName(), new PostgresqlParametersBuilder());
parametersBuilder.put(EngineType.OTHER.getName(), new OtherParametersBuilder());
}
public static DbParametersBuilder get(String engineType) {

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java Normal file
View File

@@ -0,0 +1,36 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class OtherParametersBuilder implements DbParametersBuilder {
@Override
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setComment("链接");
host.setName("url");
host.setPlaceholder("请输入链接");
databaseParameters.add(host);
DatabaseParameter userName = new DatabaseParameter();
userName.setComment("用户名");
userName.setName("username");
userName.setPlaceholder("请输入用户名");
databaseParameters.add(userName);
DatabaseParameter password = new DatabaseParameter();
password.setComment("密码");
password.setName("password");
password.setPlaceholder("请输入密码");
databaseParameters.add(password);
return databaseParameters;
}
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -14,10 +15,12 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
@@ -76,22 +79,27 @@ public class DatabaseController {
return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
}
@RequestMapping("/getDbNames/{id}")
public SemanticQueryResp getDbNames(@PathVariable("id") Long id) {
return databaseService.getDbNames(id);
@RequestMapping("/getDbNames")
public List<String> getDbNames(@RequestParam("id") Long databaseId) throws SQLException {
return databaseService.getDbNames(databaseId);
}
@RequestMapping("/getTables/{id}/{db}")
public SemanticQueryResp getTables(@PathVariable("id") Long id,
@PathVariable("db") String db) {
return databaseService.getTables(id, db);
@RequestMapping("/getTables")
public List<String> getTables(@RequestParam("databaseId") Long databaseId,
@RequestParam("db") String db) throws SQLException {
return databaseService.getTables(databaseId, db);
}
@RequestMapping("/getColumns/{id}/{db}/{table}")
public SemanticQueryResp getColumns(@PathVariable("id") Long id,
@PathVariable("db") String db,
@PathVariable("table") String table) {
return databaseService.getColumns(id, db, table);
@RequestMapping("/getColumnsByName")
public List<DBColumn> getColumnsByName(@RequestParam("databaseId") Long databaseId, @RequestParam("db") String db,
@RequestParam("table") String table) throws SQLException {
return databaseService.getColumns(databaseId, db, table);
}
@RequestMapping("/getColumnsBySql")
public List<DBColumn> getColumnsBySql(@RequestParam("databaseId") Long databaseId,
@RequestParam("sql") String sql) throws SQLException {
return databaseService.getColumns(databaseId, sql);
}
@GetMapping("/getDatabaseParameters")

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
View File

@@ -1,11 +1,14 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
@@ -30,9 +33,11 @@ public interface DatabaseService {
void deleteDatabase(Long databaseId);
SemanticQueryResp getDbNames(Long id);
List<String> getDbNames(Long id) throws SQLException;
SemanticQueryResp getTables(Long id, String db);
List<String> getTables(Long id, String db) throws SQLException;
SemanticQueryResp getColumns(Long id, String db, String table);
List<DBColumn> getColumns(Long id, String db, String table) throws SQLException;
List<DBColumn> getColumns(Long id, String sql) throws SQLException;
}

99
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -3,10 +3,11 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Pair;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -27,13 +28,16 @@ import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.util.TablesNamesFinder;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -134,9 +138,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
checkPermission(databaseResp, user);
String sql = sqlExecuteReq.getSql();
sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList());
SemanticQueryResp semanticQueryResp = executeSql(sql, databaseResp);
fillColumnComment(sql, databaseResp, semanticQueryResp);
return semanticQueryResp;
return executeSql(sql, databaseResp);
}
@Override
@@ -144,30 +146,6 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
return queryWithColumns(sql, DatabaseConverter.convert(databaseResp));
}
private void fillColumnComment(String sql, DatabaseResp databaseResp,
SemanticQueryResp semanticQueryResp) {
Pair<String, String> dbTableName = getDbTableName(sql, databaseResp);
String db = dbTableName.first;
String table = dbTableName.second;
if (StringUtils.isBlank(db) || StringUtils.isBlank(table)) {
return;
}
SemanticQueryResp columnsWithComment = getColumns(databaseResp, db, table);
Map<String, String> columnCommentMap = getColumnCommentMap(columnsWithComment.getResultList());
List<QueryColumn> columns = semanticQueryResp.getColumns();
for (QueryColumn column : columns) {
column.setComment(columnCommentMap.get(column.getNameEn()));
}
}
private Map<String, String> getColumnCommentMap(List<Map<String, Object>> resultList) {
Map<String, String> map = new HashMap<>();
for (Map<String, Object> result : resultList) {
map.put(String.valueOf(result.get("name")), String.valueOf(result.get("comment")));
}
return map;
}
private Pair<String, String> getDbTableName(String sql, DatabaseResp databaseResp) {
String dbTableName = SqlSelectHelper.getDbTableName(sql);
if (StringUtils.isBlank(dbTableName)) {
@@ -201,36 +179,63 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
@Override
public SemanticQueryResp getDbNames(Long id) {
public List<String> getDbNames(Long id) throws SQLException {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl();
return queryWithColumns(metaQueryTpl, DatabaseConverter.convert(databaseResp));
DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
return dbAdaptor.getDBs(DatabaseConverter.getConnectInfo(databaseResp));
}
@Override
public SemanticQueryResp getTables(Long id, String db) {
public List<String> getTables(Long id, String db) throws SQLException {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db);
return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
DbAdaptor dbAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
return dbAdaptor.getTables(DatabaseConverter.getConnectInfo(databaseResp), db);
}
@Override
public SemanticQueryResp getColumns(Long id, String db, String table) {
public List<DBColumn> getColumns(Long id, String db, String table) throws SQLException {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db, table);
return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
return getColumns(databaseResp, db, table);
}
public SemanticQueryResp getColumns(DatabaseResp databaseResp, String db, String table) {
public List<DBColumn> getColumns(DatabaseResp databaseResp, String db, String table) throws SQLException {
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db, table);
return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
return engineAdaptor.getColumns(DatabaseConverter.getConnectInfo(databaseResp), db, table);
}
@Override
public List<DBColumn> getColumns(Long id, String sql) throws SQLException {
String wrapSql = String.format("select * from (%s) a limit 1", sql);
DatabaseResp databaseResp = getDatabase(id);
SemanticQueryResp semanticQueryResp = executeSql(wrapSql, databaseResp);
List<DBColumn> dbColumns = Lists.newArrayList();
for (QueryColumn queryColumn : semanticQueryResp.getColumns()) {
DBColumn dbColumn = new DBColumn();
dbColumn.setColumnName(queryColumn.getNameEn());
dbColumn.setDataType(queryColumn.getType());
dbColumns.add(dbColumn);
}
return dbColumns;
}
public static void main(String[] args) {
try {
String sql = "SELECT * FROM mydatabase.mytable JOIN otherdatabase.othertable ON mytable.id = othertable.id";
// 解析SQL语句
Statement statement = CCJSqlParserUtil.parse(sql);
// 提取库表名
TablesNamesFinder tablesNamesFinder = new TablesNamesFinder();
List<String> tableNames = tablesNamesFinder.getTableList(statement);
// 打印库表名
for (String tableName : tableNames) {
System.out.println("Table Name: " + tableName);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private void checkPermission(DatabaseResp databaseResp, User user) {

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java
View File

@@ -15,11 +15,7 @@ public class DatabaseConverter {
public static Database convert(DatabaseResp databaseResp) {
Database database = new Database();
BeanUtils.copyProperties(databaseResp, database);
ConnectInfo connectInfo = new ConnectInfo();
connectInfo.setUserName(databaseResp.getUsername());
connectInfo.setPassword(databaseResp.getPassword());
connectInfo.setUrl(databaseResp.getUrl());
connectInfo.setDatabase(databaseResp.getDatabase());
ConnectInfo connectInfo = getConnectInfo(databaseResp);
database.setConnectInfo(connectInfo);
database.setVersion(databaseResp.getVersion());
return database;
@@ -81,4 +77,13 @@ public class DatabaseConverter {
return databaseResp;
}
public static ConnectInfo getConnectInfo(DatabaseResp databaseResp) {
ConnectInfo connectInfo = new ConnectInfo();
connectInfo.setUserName(databaseResp.getUsername());
connectInfo.setPassword(databaseResp.getPassword());
connectInfo.setUrl(databaseResp.getUrl());
connectInfo.setDatabase(databaseResp.getDatabase());
return connectInfo;
}
}