topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Extend support for PostgreSQL data source. (#635)

Browse Source
This commit is contained in:
lexluo09
2024-01-17 12:55:28 +08:00
committed by GitHub
parent 7707179faa
commit 74b89a9430
28 changed files with 391 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java
View File

@@ -15,6 +15,7 @@ public class DbAdaptorFactory {
dbAdaptorMap.put(EngineType.CLICKHOUSE.getName(), new ClickHouseAdaptor());
dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor());
dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor());
dbAdaptorMap.put(EngineType.POSTGRESQL.getName(), new PostgresqlAdaptor());
}
public static DbAdaptor getEngineAdaptor(String engineType) {

59
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java Normal file
View File

@@ -0,0 +1,59 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import java.util.HashMap;
import java.util.Map;
public class PostgresqlAdaptor extends DbAdaptor {
@Override
public String getDateFormat(String dateType, String dateFormat, String column) {
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "formatDateTime(toDate(parseDateTimeBestEffort(toString(%s))),'%Y-%m')".replace("%s", column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column);
} else {
return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column);
}
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "formatDateTime(toDate(%s),'%Y-%m')".replace("%s", column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "toMonday(toDate(%s))".replace("%s", column);
} else {
return column;
}
}
return column;
}
@Override
public String getDbMetaQueryTpl() {
return " SELECT datname as name FROM pg_database WHERE datistemplate = false ;";
}
@Override
public String getTableMetaQueryTpl() {
return " SELECT table_name as name FROM information_schema.tables WHERE "
+ "table_schema = 'public' AND table_catalog = '%s' ; ";
}
@Override
public String functionNameCorrector(String sql) {
Map<String, String> functionMap = new HashMap<>();
functionMap.put("MONTH".toLowerCase(), "toMonth");
functionMap.put("DAY".toLowerCase(), "toDayOfMonth");
functionMap.put("YEAR".toLowerCase(), "toYear");
return SqlParserReplaceHelper.replaceFunction(sql, functionMap);
}
@Override
public String getColumnMetaQueryTpl() {
return " SELECT column_name as name, data_type as dataType FROM information_schema.columns "
+ "WHERE table_schema = 'public' AND table_catalog = '%s' AND table_name = '%s' ; ";
}
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java
View File

@@ -37,6 +37,7 @@ public class ModelYamlManager {
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
addInterCntMetric(modelResp.getBizName(), modelDetail);
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java
View File

@@ -16,6 +16,8 @@ public class DataSource {
private Long sourceId;
private String type;
private String sqlQuery;
private String tableQuery;

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SemanticSqlDialect.java
View File

@@ -18,7 +18,6 @@ public class SemanticSqlDialect extends SqlDialect {
.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
.withLiteralQuoteString("'")
.withLiteralEscapedQuoteString("''")
.withIdentifierQuoteString("`")
.withUnquotedCasing(Casing.UNCHANGED)
.withQuotedCasing(Casing.UNCHANGED)
.withCaseSensitive(false);

9
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
@@ -43,7 +44,13 @@ public class DataSourceNode extends SemanticNode {
if (datasource.getSqlQuery() != null && !datasource.getSqlQuery().isEmpty()) {
sqlTable = datasource.getSqlQuery();
} else if (datasource.getTableQuery() != null && !datasource.getTableQuery().isEmpty()) {
sqlTable = "select * from " + datasource.getTableQuery();
if (datasource.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
String fullTableName = Arrays.stream(datasource.getTableQuery().split("\\."))
.collect(Collectors.joining(".public."));
sqlTable = "select * from " + fullTableName;
} else {
sqlTable = "select * from " + datasource.getTableQuery();
}
}
if (sqlTable.isEmpty()) {
throw new Exception("DatasourceNode build error [tableSqlNode not found]");

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java
View File

@@ -338,6 +338,7 @@ public abstract class SemanticNode {
public static SqlNode optimize(SqlValidatorScope scope, HeadlessSchema schema, SqlNode sqlNode) {
try {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
hepProgramBuilder.addRuleInstance(new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema));
RelOptPlanner relOptPlanner = new HepPlanner(hepProgramBuilder.build());
RelToSqlConverter converter = new RelToSqlConverter(SemanticSqlDialect.DEFAULT);

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java
View File

@@ -15,6 +15,8 @@ public class DataModelYamlTpl {
private Long sourceId;
private String type;
private String sqlQuery;
private String tableQuery;

36
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java
View File

@@ -1,23 +1,5 @@
package com.tencent.supersonic.headless.core.utils;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import static com.tencent.supersonic.common.pojo.Constants.COLON;
import static com.tencent.supersonic.common.pojo.Constants.DOUBLE_SLASH;
@@ -27,6 +9,23 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR;
import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import javax.sql.DataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
* tools functions about jdbc
*/
@@ -127,7 +126,6 @@ public class JdbcDataSourceUtils {
if (dataTypeEnum != null) {
return dataTypeEnum.getDriver();
}
throw new RuntimeException("Not supported data type: jdbcUrl=" + jdbcUrl);
}