(improvement)(headless) Remove ItemDateResp and databaseResp from SemanticModel. (#648)

This commit is contained in:
lexluo09
2024-01-18 16:30:18 +08:00
committed by GitHub
parent dfb8e3a427
commit a06a1fa898
13 changed files with 109 additions and 93 deletions

View File

@@ -16,10 +16,10 @@ public class LLMParserConfig {
@Value("${query2sql.path:/query2sql}")
private String queryToSqlPath;
@Value("${dimension.topn:5}")
@Value("${dimension.topn:10}")
private Integer dimensionTopN;
@Value("${metric.topn:5}")
@Value("${metric.topn:10}")
private Integer metricTopN;
@Value("${all.model:false}")

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import lombok.extern.slf4j.Slf4j;
@@ -30,10 +30,10 @@ public class JdbcExecutor implements QueryExecutor {
return null;
}
log.info("query SQL: {}", queryStatement.getSql());
DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp();
log.info("database info:{}", databaseResp);
Database database = queryStatement.getSemanticModel().getDatabase();
log.info("database info:{}", database);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);
SqlUtils sqlUtils = this.sqlUtils.init(database);
sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns);
return queryResultWithColumns;
}

View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -19,8 +18,7 @@ public class SemanticModel {
private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations;
private ItemDateResp dataDate;
private DatabaseResp databaseResp;
private Database database;
public List<Dimension> getDimensions() {
return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());

View File

@@ -10,19 +10,18 @@ import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* supplement the QueryStatement when query with custom aggregation method
@@ -57,8 +56,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getSemanticModel().getDataDate());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
@@ -109,9 +107,9 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp();
Database database = queryStatement.getSemanticModel().getDatabase();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
EngineType.valueOf(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath());
@@ -145,7 +143,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getSemanticModel().getDataDate());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);

View File

@@ -29,7 +29,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
private final CalculateAggConverter calculateConverterAgg;
public ParserDefaultConverter(CalculateAggConverter calculateConverterAgg,
SqlGenerateUtils sqlGenerateUtils) {
SqlGenerateUtils sqlGenerateUtils) {
this.calculateConverterAgg = calculateConverterAgg;
this.sqlGenerateUtils = sqlGenerateUtils;
}
@@ -47,8 +47,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq,
queryStatement.getSemanticModel().getDataDate()));
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, null));
BeanUtils.copyProperties(metricReq, metricQueryReq);
}
@@ -56,8 +55,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
MetricQueryReq metricQueryReq = new MetricQueryReq();
metricQueryReq.setMetrics(queryStructReq.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getSemanticModel().getDataDate());
String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where);

View File

@@ -3,14 +3,19 @@ package com.tencent.supersonic.headless.core.pojo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import java.util.List;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class Database extends RecordInfo {
private Long id;
private Long domainId;
@@ -21,6 +26,15 @@ public class Database extends RecordInfo {
private String version;
private String url;
private String username;
private String password;
private String database;
private String schema;
/**
* mysql,clickhouse
*/

View File

@@ -1,20 +1,13 @@
package com.tencent.supersonic.headless.core.pojo;
import static com.tencent.supersonic.common.pojo.Constants.STATISTIC;
import com.alibaba.druid.filter.Filter;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.wall.WallConfig;
import com.alibaba.druid.wall.WallFilter;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
@@ -22,8 +15,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static com.tencent.supersonic.common.pojo.Constants.STATISTIC;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@@ -154,9 +152,9 @@ public class JdbcDataSource {
}
}
public void removeDatasource(DatabaseResp jdbcSourceInfo) {
public void removeDatasource(Database database) {
String key = getDataSourceKey(jdbcSourceInfo);
String key = getDataSourceKey(database);
Lock lock = getDataSourceLock(key);
@@ -176,15 +174,15 @@ public class JdbcDataSource {
}
}
public DruidDataSource getDataSource(DatabaseResp jdbcSourceInfo) throws RuntimeException {
public DruidDataSource getDataSource(Database database) throws RuntimeException {
String name = jdbcSourceInfo.getName();
String type = jdbcSourceInfo.getType();
String jdbcUrl = jdbcSourceInfo.getUrl();
String username = jdbcSourceInfo.getUsername();
String password = jdbcSourceInfo.getPassword();
String name = database.getName();
String type = database.getType();
String jdbcUrl = database.getUrl();
String username = database.getUsername();
String password = database.getPassword();
String key = getDataSourceKey(jdbcSourceInfo);
String key = getDataSourceKey(database);
DruidDataSource druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
@@ -306,10 +304,10 @@ public class JdbcDataSource {
return druidDataSource;
}
private String getDataSourceKey(DatabaseResp jdbcSourceInfo) {
return JdbcDataSourceUtils.getKey(jdbcSourceInfo.getName(),
jdbcSourceInfo.getUrl(),
jdbcSourceInfo.getUsername(),
jdbcSourceInfo.getPassword(), "", false);
private String getDataSourceKey(Database database) {
return JdbcDataSourceUtils.getKey(database.getName(),
database.getUrl(),
database.getUsername(),
database.getPassword(), "", false);
}
}

View File

@@ -12,7 +12,6 @@ import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.sql.Connection;
@@ -147,27 +146,27 @@ public class JdbcDataSourceUtils {
return MD5Util.getMD5(sb.toString(), true, 64);
}
public DataSource getDataSource(DatabaseResp databaseResp) throws RuntimeException {
return jdbcDataSource.getDataSource(databaseResp);
public DataSource getDataSource(Database database) throws RuntimeException {
return jdbcDataSource.getDataSource(database);
}
public Connection getConnection(DatabaseResp databaseResp) throws RuntimeException {
Connection conn = getConnectionWithRetry(databaseResp);
public Connection getConnection(Database database) throws RuntimeException {
Connection conn = getConnectionWithRetry(database);
if (conn == null) {
try {
releaseDataSource(databaseResp);
DataSource dataSource = getDataSource(databaseResp);
releaseDataSource(database);
DataSource dataSource = getDataSource(database);
return dataSource.getConnection();
} catch (Exception e) {
log.error("Get connection error, jdbcUrl:{}, e:{}", databaseResp.getUrl(), e);
throw new RuntimeException("Get connection error, jdbcUrl:" + databaseResp.getUrl()
log.error("Get connection error, jdbcUrl:{}, e:{}", database.getUrl(), e);
throw new RuntimeException("Get connection error, jdbcUrl:" + database.getUrl()
+ " you can try again later or reset datasource");
}
}
return conn;
}
private Connection getConnectionWithRetry(DatabaseResp databaseResp) {
private Connection getConnectionWithRetry(Database database) {
int rc = 1;
for (; ; ) {
@@ -176,7 +175,7 @@ public class JdbcDataSourceUtils {
}
try {
Connection connection = getDataSource(databaseResp).getConnection();
Connection connection = getDataSource(database).getConnection();
if (connection != null && connection.isValid(5)) {
return connection;
}
@@ -194,7 +193,7 @@ public class JdbcDataSourceUtils {
}
}
public void releaseDataSource(DatabaseResp databaseResp) {
jdbcDataSource.removeDatasource(databaseResp);
public void releaseDataSource(Database database) {
jdbcDataSource.removeDatasource(database);
}
}

View File

@@ -5,8 +5,8 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.rmi.ServerException;
import java.sql.Connection;
@@ -37,7 +37,7 @@ import org.springframework.stereotype.Component;
public class SqlUtils {
@Getter
private DatabaseResp databaseResp;
private Database database;
@Autowired
private JdbcDataSource jdbcDataSource;
@@ -58,20 +58,20 @@ public class SqlUtils {
}
public SqlUtils(DatabaseResp databaseResp) {
this.databaseResp = databaseResp;
this.dataTypeEnum = DataType.urlOf(databaseResp.getUrl());
public SqlUtils(Database database) {
this.database = database;
this.dataTypeEnum = DataType.urlOf(database.getUrl());
}
public SqlUtils init(DatabaseResp databaseResp) {
public SqlUtils init(Database database) {
//todo Password decryption
return SqlUtilsBuilder
.getBuilder()
.withName(databaseResp.getId() + AT_SYMBOL + databaseResp.getName())
.withType(databaseResp.getType())
.withJdbcUrl(databaseResp.getUrl())
.withUsername(databaseResp.getUsername())
.withPassword(databaseResp.getPassword())
.withName(database.getId() + AT_SYMBOL + database.getName())
.withType(database.getType())
.withJdbcUrl(database.getUrl())
.withUsername(database.getUsername())
.withPassword(database.getPassword())
.withJdbcDataSource(this.jdbcDataSource)
.withResultLimit(this.resultLimit)
.withIsQueryLogEnable(this.isQueryLogEnable)
@@ -96,15 +96,15 @@ public class SqlUtils {
public JdbcTemplate jdbcTemplate() throws RuntimeException {
Connection connection = null;
try {
connection = jdbcDataSourceUtils.getConnection(databaseResp);
connection = jdbcDataSourceUtils.getConnection(database);
} catch (Exception e) {
log.warn("e:", e);
} finally {
JdbcDataSourceUtils.releaseConnection(connection);
}
DataSource dataSource = jdbcDataSourceUtils.getDataSource(databaseResp);
DataSource dataSource = jdbcDataSourceUtils.getDataSource(database);
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
jdbcTemplate.setDatabaseProductName(databaseResp.getName());
jdbcTemplate.setDatabaseProductName(database.getName());
jdbcTemplate.setFetchSize(500);
log.info("jdbcTemplate:{}, dataSource:{}", jdbcTemplate, dataSource);
return jdbcTemplate;
@@ -234,7 +234,7 @@ public class SqlUtils {
}
public SqlUtils build() {
DatabaseResp databaseResp = DatabaseResp.builder()
Database database = Database.builder()
.name(this.name)
.type(this.type)
.url(this.jdbcUrl)
@@ -242,7 +242,7 @@ public class SqlUtils {
.password(this.password)
.build();
SqlUtils sqlUtils = new SqlUtils(databaseResp);
SqlUtils sqlUtils = new SqlUtils(database);
sqlUtils.jdbcDataSource = this.jdbcDataSource;
sqlUtils.resultLimit = this.resultLimit;
sqlUtils.isQueryLogEnable = this.isQueryLogEnable;

View File

@@ -27,6 +27,7 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
@@ -77,7 +78,7 @@ public class HeadlessSchemaManager {
Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
semanticModel.setDatabaseResp(databaseResp);
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));

View File

@@ -133,7 +133,7 @@ public class DatabaseServiceImpl implements DatabaseService {
@Override
public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) {
return queryWithColumns(sql, databaseResp);
return queryWithColumns(sql, DatabaseConverter.convert(databaseResp));
}
@Override
@@ -143,9 +143,9 @@ public class DatabaseServiceImpl implements DatabaseService {
LinkedHashMap::putAll);
}
private QueryResultWithSchemaResp queryWithColumns(String sql, DatabaseResp databaseResp) {
private QueryResultWithSchemaResp queryWithColumns(String sql, Database database) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);
SqlUtils sqlUtils = this.sqlUtils.init(database);
log.info("query SQL: {}", sql);
sqlUtils.queryInternal(sql, queryResultWithColumns);
return queryResultWithColumns;
@@ -160,7 +160,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl();
return queryWithColumns(metaQueryTpl, databaseResp);
return queryWithColumns(metaQueryTpl, DatabaseConverter.convert(databaseResp));
}
@Override
@@ -169,7 +169,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db);
return queryWithColumns(metaQuerySql, databaseResp);
return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
}
@Override
@@ -178,7 +178,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db, table);
return queryWithColumns(metaQuerySql, databaseResp);
return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
}
}

View File

@@ -1,16 +1,15 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
@@ -98,10 +97,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq);
semanticModel.setDataDate(itemDateResp);
return semanticModel;
return headlessSchemaManager.get(queryStructReq.getModelIdStr());
}
}

View File

@@ -9,8 +9,22 @@ import com.tencent.supersonic.headless.core.pojo.Database;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;
public class DatabaseConverter {
public static Database convert(DatabaseResp databaseResp) {
Database database = new Database();
BeanUtils.copyProperties(databaseResp, database);
ConnectInfo connectInfo = new ConnectInfo();
connectInfo.setUserName(databaseResp.getUsername());
connectInfo.setPassword(databaseResp.getPassword());
connectInfo.setUrl(databaseResp.getUrl());
connectInfo.setDatabase(databaseResp.getDatabase());
database.setConnectInfo(connectInfo);
database.setVersion(databaseResp.getVersion());
return database;
}
public static Database convert(DatabaseReq databaseReq) {
Database database = new Database();
BeanUtils.copyProperties(databaseReq, database);