(improvement)(chat) Extend support for PostgreSQL data source. (#635)

This commit is contained in:
lexluo09
2024-01-17 12:55:28 +08:00
committed by GitHub
parent 7707179faa
commit 74b89a9430
28 changed files with 391 additions and 45 deletions

View File

@@ -1,15 +1,10 @@
package com.tencent.supersonic.common.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.ToString;
/**
* @author: kanedai
* @date: 2023/3/29
*/
@Data
@AllArgsConstructor
@ToString
@@ -19,5 +14,5 @@ public class ItemDateResp {
private String startDate;
private String endDate;
private String datePeriod;
private List<String> unavailableDateList = new ArrayList<>();
private List<String> unavailableDateList;
}

View File

@@ -40,7 +40,9 @@ public enum DataType {
IMPALA("impala", "impala", "com.cloudera.impala.jdbc41.Driver", "", "", "'", "'"),
TDENGINE("TAOS", "TAOS", "com.taosdata.jdbc.TSDBDriver", "'", "'", "\"", "\"");
TDENGINE("TAOS", "TAOS", "com.taosdata.jdbc.TSDBDriver", "'", "'", "\"", "\""),
POSTGRESQL("postgresql", "postgresql", "org.postgresql.Driver", "'", "'", "\"", "\"");
private String feature;
private String desc;

View File

@@ -8,7 +8,8 @@ public enum EngineType {
DORIS(2, "doris"),
CLICKHOUSE(3, "clickhouse"),
KAFKA(4, "kafka"),
H2(5, "h2");
H2(5, "h2"),
POSTGRESQL(6, "postgresql");
private Integer code;

View File

@@ -30,6 +30,7 @@ public class DatabaseReq {
private String description;
private String schema;
private String url;
private List<String> admins = Lists.newArrayList();

View File

@@ -40,6 +40,8 @@ public class DatabaseResp extends RecordInfo {
private String version;
private String schema;
private boolean hasPermission = false;
private boolean hasUsePermission = false;

View File

@@ -15,6 +15,7 @@ public class DbAdaptorFactory {
dbAdaptorMap.put(EngineType.CLICKHOUSE.getName(), new ClickHouseAdaptor());
dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor());
dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor());
dbAdaptorMap.put(EngineType.POSTGRESQL.getName(), new PostgresqlAdaptor());
}
public static DbAdaptor getEngineAdaptor(String engineType) {

View File

@@ -0,0 +1,59 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import java.util.HashMap;
import java.util.Map;
public class PostgresqlAdaptor extends DbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "formatDateTime(toDate(parseDateTimeBestEffort(toString(%s))),'%Y-%m')".replace("%s", column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column);
} else {
return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column);
}
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "formatDateTime(toDate(%s),'%Y-%m')".replace("%s", column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "toMonday(toDate(%s))".replace("%s", column);
} else {
return column;
}
}
return column;
}
@Override
public String getDbMetaQueryTpl() {
return " SELECT datname as name FROM pg_database WHERE datistemplate = false ;";
}
@Override
public String getTableMetaQueryTpl() {
return " SELECT table_name as name FROM information_schema.tables WHERE "
+ "table_schema = 'public' AND table_catalog = '%s' ; ";
}
@Override
public String functionNameCorrector(String sql) {
Map<String, String> functionMap = new HashMap<>();
functionMap.put("MONTH".toLowerCase(), "toMonth");
functionMap.put("DAY".toLowerCase(), "toDayOfMonth");
functionMap.put("YEAR".toLowerCase(), "toYear");
return SqlParserReplaceHelper.replaceFunction(sql, functionMap);
}
@Override
public String getColumnMetaQueryTpl() {
return " SELECT column_name as name, data_type as dataType FROM information_schema.columns "
+ "WHERE table_schema = 'public' AND table_catalog = '%s' AND table_name = '%s' ; ";
}
}

View File

@@ -37,6 +37,7 @@ public class ModelYamlManager {
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
addInterCntMetric(modelResp.getBizName(), modelDetail);
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));

View File

@@ -16,6 +16,8 @@ public class DataSource {
private Long sourceId;
private String type;
private String sqlQuery;
private String tableQuery;

View File

@@ -18,7 +18,6 @@ public class SemanticSqlDialect extends SqlDialect {
.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
.withLiteralQuoteString("'")
.withLiteralEscapedQuoteString("''")
.withIdentifierQuoteString("`")
.withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED)
.withCaseSensitive(false);

View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
@@ -43,7 +44,13 @@ public class DataSourceNode extends SemanticNode {
if (datasource.getSqlQuery() != null && !datasource.getSqlQuery().isEmpty()) {
sqlTable = datasource.getSqlQuery();
} else if (datasource.getTableQuery() != null && !datasource.getTableQuery().isEmpty()) {
sqlTable = "select * from " + datasource.getTableQuery();
if (datasource.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
String fullTableName = Arrays.stream(datasource.getTableQuery().split("\\."))
.collect(Collectors.joining(".public."));
sqlTable = "select * from " + fullTableName;
} else {
sqlTable = "select * from " + datasource.getTableQuery();
}
}
if (sqlTable.isEmpty()) {
throw new Exception("DatasourceNode build error [tableSqlNode not found]");

View File

@@ -338,6 +338,7 @@ public abstract class SemanticNode {
public static SqlNode optimize(SqlValidatorScope scope, HeadlessSchema schema, SqlNode sqlNode) {
try {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
hepProgramBuilder.addRuleInstance(new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema));
RelOptPlanner relOptPlanner = new HepPlanner(hepProgramBuilder.build());
RelToSqlConverter converter = new RelToSqlConverter(SemanticSqlDialect.DEFAULT);

View File

@@ -15,6 +15,8 @@ public class DataModelYamlTpl {
private Long sourceId;
private String type;
private String sqlQuery;
private String tableQuery;

View File

@@ -1,23 +1,5 @@
package com.tencent.supersonic.headless.core.utils;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import static com.tencent.supersonic.common.pojo.Constants.COLON;
import static com.tencent.supersonic.common.pojo.Constants.DOUBLE_SLASH;
@@ -27,6 +9,23 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR;
import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import javax.sql.DataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
* tools functions about jdbc
*/
@@ -127,7 +126,6 @@ public class JdbcDataSourceUtils {
if (dataTypeEnum != null) {
return dataTypeEnum.getDriver();
}
throw new RuntimeException("Not supported data type: jdbcUrl=" + jdbcUrl);
}

View File

@@ -110,6 +110,11 @@
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
</dependencies>
</project>

View File

@@ -120,7 +120,7 @@ public class HeadlessSchemaManager {
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
.sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));

View File

@@ -0,0 +1,44 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class ClickHouseParametersBuilder implements DbParametersBuilder {
@Override
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setName("host");
host.setEnName("host");
host.setComment("请输入host");
databaseParameters.add(host);
DatabaseParameter port = new DatabaseParameter();
port.setName("port");
port.setEnName("port");
port.setComment("请输入端口号");
databaseParameters.add(port);
DatabaseParameter userName = new DatabaseParameter();
userName.setName("用户名");
userName.setEnName("username");
databaseParameters.add(userName);
DatabaseParameter password = new DatabaseParameter();
password.setName("密码");
password.setEnName("password");
databaseParameters.add(password);
DatabaseParameter database = new DatabaseParameter();
database.setName("数据库名称");
database.setEnName("database");
databaseParameters.add(database);
return databaseParameters;
}
}

View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class DatabaseParameter {
private String name;
private String enName;
private String comment;
private String defaultValue;
}

View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.enums.EngineType;
import java.util.LinkedHashMap;
import java.util.Map;
public class DbParameterFactory {
private static Map<String, DbParametersBuilder> parametersBuilder;
static {
parametersBuilder = new LinkedHashMap<>();
parametersBuilder.put(EngineType.H2.getName(), new H2ParametersBuilder());
parametersBuilder.put(EngineType.CLICKHOUSE.getName(), new ClickHouseParametersBuilder());
parametersBuilder.put(EngineType.MYSQL.getName(), new MysqlParametersBuilder());
parametersBuilder.put(EngineType.POSTGRESQL.getName(), new PostgresqlParametersBuilder());
}
public static DbParametersBuilder get(String engineType) {
return parametersBuilder.get(engineType);
}
public static Map<String, DbParametersBuilder> getMap() {
return parametersBuilder;
}
}

View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
public interface DbParametersBuilder {
List<DatabaseParameter> build();
}

View File

@@ -0,0 +1,44 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class H2ParametersBuilder implements DbParametersBuilder {
@Override
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setName("链接");
host.setEnName("url");
host.setComment("请输入链接");
databaseParameters.add(host);
DatabaseParameter port = new DatabaseParameter();
port.setName("port");
port.setEnName("port");
port.setComment("请输入端口号");
databaseParameters.add(port);
DatabaseParameter userName = new DatabaseParameter();
userName.setName("用户名");
userName.setEnName("username");
databaseParameters.add(userName);
DatabaseParameter password = new DatabaseParameter();
password.setName("密码");
password.setEnName("password");
databaseParameters.add(password);
DatabaseParameter database = new DatabaseParameter();
database.setName("数据库名称");
database.setEnName("database");
databaseParameters.add(database);
return databaseParameters;
}
}

View File

@@ -0,0 +1,50 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class MysqlParametersBuilder implements DbParametersBuilder {
@Override
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setName("host");
host.setEnName("host");
host.setComment("请输入host");
databaseParameters.add(host);
DatabaseParameter port = new DatabaseParameter();
port.setName("port");
port.setEnName("port");
port.setComment("请输入端口号");
databaseParameters.add(port);
DatabaseParameter version = new DatabaseParameter();
version.setName("数据库版本");
version.setEnName("version");
version.setComment("请输入数据库版本");
databaseParameters.add(version);
DatabaseParameter userName = new DatabaseParameter();
userName.setName("用户名");
userName.setEnName("username");
databaseParameters.add(userName);
DatabaseParameter password = new DatabaseParameter();
password.setName("密码");
password.setEnName("password");
databaseParameters.add(password);
DatabaseParameter database = new DatabaseParameter();
database.setName("数据库名称");
database.setEnName("database");
databaseParameters.add(database);
return databaseParameters;
}
}

View File

@@ -0,0 +1,50 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class PostgresqlParametersBuilder implements DbParametersBuilder {
@Override
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setName("host");
host.setEnName("host");
host.setComment("请输入host");
databaseParameters.add(host);
DatabaseParameter port = new DatabaseParameter();
port.setName("port");
port.setEnName("port");
port.setComment("请输入端口号");
databaseParameters.add(port);
DatabaseParameter userName = new DatabaseParameter();
userName.setName("用户名");
userName.setEnName("username");
databaseParameters.add(userName);
DatabaseParameter password = new DatabaseParameter();
password.setName("密码");
password.setEnName("password");
databaseParameters.add(password);
DatabaseParameter schema = new DatabaseParameter();
schema.setName("schema");
schema.setEnName("schema");
schema.setDefaultValue("public");
databaseParameters.add(schema);
DatabaseParameter database = new DatabaseParameter();
database.setName("数据库名称");
database.setEnName("database");
databaseParameters.add(database);
return databaseParameters;
}
}

View File

@@ -6,7 +6,9 @@ import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import java.util.Map;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -53,7 +55,7 @@ public class DatabaseController {
@GetMapping("/getDatabaseList")
public List<DatabaseResp> getDatabaseList(HttpServletRequest request,
HttpServletResponse response) {
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.getDatabaseList(user);
}
@@ -66,8 +68,8 @@ public class DatabaseController {
@PostMapping("/executeSql")
public QueryResultWithSchemaResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.executeSql(sqlExecuteReq.getSql(), sqlExecuteReq.getId(), user);
}
@@ -90,4 +92,10 @@ public class DatabaseController {
return databaseService.getColumns(id, db, table);
}
@GetMapping("/getDatabaseParameters")
public Map<String, List<DatabaseParameter>> getDatabaseParameters(HttpServletRequest request,
HttpServletResponse response) {
return databaseService.getDatabaseParameters();
}
}

View File

@@ -4,8 +4,9 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import java.util.List;
import java.util.Map;
public interface DatabaseService {
@@ -14,6 +15,8 @@ public interface DatabaseService {
QueryResultWithSchemaResp executeSql(String sql, Long id, User user);
Map<String, List<DatabaseParameter>> getDatabaseParameters();
boolean testConnect(DatabaseReq databaseReq, User user);
DatabaseResp createOrUpdateDatabase(DatabaseReq databaseReq, User user);

View File

@@ -7,23 +7,26 @@ import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import com.tencent.supersonic.headless.server.pojo.DbParameterFactory;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Service
@@ -34,8 +37,8 @@ public class DatabaseServiceImpl implements DatabaseService {
private ModelService datasourceService;
public DatabaseServiceImpl(DatabaseRepository databaseRepository,
SqlUtils sqlUtils,
@Lazy ModelService datasourceService) {
SqlUtils sqlUtils,
@Lazy ModelService datasourceService) {
this.databaseRepository = databaseRepository;
this.sqlUtils = sqlUtils;
this.datasourceService = datasourceService;
@@ -67,8 +70,8 @@ public class DatabaseServiceImpl implements DatabaseService {
public List<DatabaseResp> getDatabaseList(User user) {
List<DatabaseResp> databaseResps =
databaseRepository.getDatabaseList()
.stream().map(DatabaseConverter::convert)
.collect(Collectors.toList());
.stream().map(DatabaseConverter::convert)
.collect(Collectors.toList());
fillPermission(databaseResps, user);
return databaseResps;
}
@@ -133,6 +136,13 @@ public class DatabaseServiceImpl implements DatabaseService {
return queryWithColumns(sql, databaseResp);
}
@Override
public Map<String, List<DatabaseParameter>> getDatabaseParameters() {
return DbParameterFactory.getMap().entrySet().stream().collect(LinkedHashMap::new,
(map, entry) -> map.put(entry.getKey(), entry.getValue().build()),
LinkedHashMap::putAll);
}
private QueryResultWithSchemaResp queryWithColumns(String sql, DatabaseResp databaseResp) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);

View File

@@ -83,4 +83,8 @@ logging:
inMemoryEmbeddingStore:
persistent:
path: /tmp
path: /tmp
query:
optimizer:
enable: true

View File

@@ -73,6 +73,7 @@
<easyexcel.version>2.2.6</easyexcel.version>
<poi.version>3.17</poi.version>
<langchain4j.version>0.24.0</langchain4j.version>
<postgresql.version>42.7.1</postgresql.version>
</properties>
<dependencyManagement>