topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Remove ItemDateResp and databaseResp from SemanticModel. (#648)

Browse Source
This commit is contained in:
lexluo09
2024-01-18 16:30:18 +08:00
committed by GitHub
parent dfb8e3a427
commit a06a1fa898
13 changed files with 109 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/core/src/main/java/com/tencent/supersonic/chat/core/config/LLMParserConfig.java
View File

@@ -16,10 +16,10 @@ public class LLMParserConfig {
@Value("${query2sql.path:/query2sql}") @Value("${query2sql.path:/query2sql}")
private String queryToSqlPath; private String queryToSqlPath;
@Value("${dimension.topn:5}") @Value("${dimension.topn:10}")
private Integer dimensionTopN; private Integer dimensionTopN;
@Value("${metric.topn:5}") @Value("${metric.topn:10}")
private Integer metricTopN; private Integer metricTopN;
@Value("${all.model:false}") @Value("${all.model:false}")

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.executor; package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlUtils; import com.tencent.supersonic.headless.core.utils.SqlUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -30,10 +30,10 @@ public class JdbcExecutor implements QueryExecutor {
return null; return null;
} }
log.info("query SQL: {}", queryStatement.getSql()); log.info("query SQL: {}", queryStatement.getSql());
DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp(); Database database = queryStatement.getSemanticModel().getDatabase();
log.info("database info:{}", databaseResp); log.info("database info:{}", database);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); SqlUtils sqlUtils = this.sqlUtils.init(database);
sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns); sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns);
return queryResultWithColumns; return queryResultWithColumns;
} }

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/SemanticModel.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.parser.calcite.s2sql; package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@@ -19,8 +18,7 @@ public class SemanticModel {
private Map<String, List<Dimension>> dimensionMap = new HashMap<>(); private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>(); private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
private ItemDateResp dataDate; private Database database;
private DatabaseResp databaseResp;
public List<Dimension> getDimensions() { public List<Dimension> getDimensions() {
return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -10,19 +10,18 @@ import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/** /**
* supplement the QueryStatement when query with custom aggregation method * supplement the QueryStatement when query with custom aggregation method
@@ -57,8 +56,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setAlias(metricTableName); metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
queryStatement.getSemanticModel().getDataDate());
log.info("in generateSqlCommand, complete where:{}", where); log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);
@@ -109,9 +107,9 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq(); ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
DatabaseResp databaseResp = queryStatement.getSemanticModel().getDatabaseResp(); Database database = queryStatement.getSemanticModel().getDatabase();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement, ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion()); EngineType.valueOf(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(parseSqlReq.getSql()); sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables()); sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath()); sqlCommend.setRootPath(parseSqlReq.getRootPath());
@@ -145,7 +143,7 @@ public class CalculateAggConverter implements HeadlessConverter {
metricTable.setAlias(metricTableName); metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructReq.getMetrics()); metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups()); metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getSemanticModel().getDataDate()); String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where); metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION); metricTable.setAggOption(AggOption.AGGREGATION);

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
View File

@@ -47,8 +47,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
MetricQueryReq metricQueryReq = queryStatement.getMetricReq(); MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement); MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq, null));
queryStatement.getSemanticModel().getDataDate()));
BeanUtils.copyProperties(metricReq, metricQueryReq); BeanUtils.copyProperties(metricReq, metricQueryReq);
} }
@@ -56,8 +55,7 @@ public class ParserDefaultConverter implements HeadlessConverter {
MetricQueryReq metricQueryReq = new MetricQueryReq(); MetricQueryReq metricQueryReq = new MetricQueryReq();
metricQueryReq.setMetrics(queryStructReq.getMetrics()); metricQueryReq.setMetrics(queryStructReq.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups()); metricQueryReq.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, String where = sqlGenerateUtils.generateWhere(queryStructReq, null);
queryStatement.getSemanticModel().getDataDate());
log.info("in generateSqlCommend, complete where:{}", where); log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where); metricQueryReq.setWhere(where);

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java
View File

@@ -3,14 +3,19 @@ package com.tencent.supersonic.headless.core.pojo;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import lombok.NoArgsConstructor;
@Data @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class Database extends RecordInfo { public class Database extends RecordInfo {
private Long id; private Long id;
private Long domainId; private Long domainId;
@@ -21,6 +26,15 @@ public class Database extends RecordInfo {
private String version; private String version;
private String url;
private String username;
private String password;
private String database;
private String schema;
/** /**
* mysql,clickhouse * mysql,clickhouse
*/ */

48
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java
View File

@@ -1,20 +1,13 @@
package com.tencent.supersonic.headless.core.pojo; package com.tencent.supersonic.headless.core.pojo;
import static com.tencent.supersonic.common.pojo.Constants.STATISTIC;
import com.alibaba.druid.filter.Filter; import com.alibaba.druid.filter.Filter;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.wall.WallConfig; import com.alibaba.druid.wall.WallConfig;
import com.alibaba.druid.wall.WallFilter; import com.alibaba.druid.wall.WallFilter;
import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import java.util.Arrays; import java.util.Arrays;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
@@ -22,8 +15,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import lombok.Getter;
import static com.tencent.supersonic.common.pojo.Constants.STATISTIC; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
@Slf4j @Slf4j
@Component @Component
@@ -154,9 +152,9 @@ public class JdbcDataSource {
} }
} }
public void removeDatasource(DatabaseResp jdbcSourceInfo) { public void removeDatasource(Database database) {
String key = getDataSourceKey(jdbcSourceInfo); String key = getDataSourceKey(database);
Lock lock = getDataSourceLock(key); Lock lock = getDataSourceLock(key);
@@ -176,15 +174,15 @@ public class JdbcDataSource {
} }
} }
public DruidDataSource getDataSource(DatabaseResp jdbcSourceInfo) throws RuntimeException { public DruidDataSource getDataSource(Database database) throws RuntimeException {
String name = jdbcSourceInfo.getName(); String name = database.getName();
String type = jdbcSourceInfo.getType(); String type = database.getType();
String jdbcUrl = jdbcSourceInfo.getUrl(); String jdbcUrl = database.getUrl();
String username = jdbcSourceInfo.getUsername(); String username = database.getUsername();
String password = jdbcSourceInfo.getPassword(); String password = database.getPassword();
String key = getDataSourceKey(jdbcSourceInfo); String key = getDataSourceKey(database);
DruidDataSource druidDataSource = dataSourceMap.get(key); DruidDataSource druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) { if (druidDataSource != null && !druidDataSource.isClosed()) {
@@ -306,10 +304,10 @@ public class JdbcDataSource {
return druidDataSource; return druidDataSource;
} }
private String getDataSourceKey(DatabaseResp jdbcSourceInfo) { private String getDataSourceKey(Database database) {
return JdbcDataSourceUtils.getKey(jdbcSourceInfo.getName(), return JdbcDataSourceUtils.getKey(database.getName(),
jdbcSourceInfo.getUrl(), database.getUrl(),
jdbcSourceInfo.getUsername(), database.getUsername(),
jdbcSourceInfo.getPassword(), "", false); database.getPassword(), "", false);
} }
} }

25
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java
View File

@@ -12,7 +12,6 @@ import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import com.alibaba.druid.util.StringUtils; import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util; import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.sql.Connection; import java.sql.Connection;
@@ -147,27 +146,27 @@ public class JdbcDataSourceUtils {
return MD5Util.getMD5(sb.toString(), true, 64); return MD5Util.getMD5(sb.toString(), true, 64);
} }
public DataSource getDataSource(DatabaseResp databaseResp) throws RuntimeException { public DataSource getDataSource(Database database) throws RuntimeException {
return jdbcDataSource.getDataSource(databaseResp); return jdbcDataSource.getDataSource(database);
} }
public Connection getConnection(DatabaseResp databaseResp) throws RuntimeException { public Connection getConnection(Database database) throws RuntimeException {
Connection conn = getConnectionWithRetry(databaseResp); Connection conn = getConnectionWithRetry(database);
if (conn == null) { if (conn == null) {
try { try {
releaseDataSource(databaseResp); releaseDataSource(database);
DataSource dataSource = getDataSource(databaseResp); DataSource dataSource = getDataSource(database);
return dataSource.getConnection(); return dataSource.getConnection();
} catch (Exception e) { } catch (Exception e) {
log.error("Get connection error, jdbcUrl:{}, e:{}", databaseResp.getUrl(), e); log.error("Get connection error, jdbcUrl:{}, e:{}", database.getUrl(), e);
throw new RuntimeException("Get connection error, jdbcUrl:" + databaseResp.getUrl() throw new RuntimeException("Get connection error, jdbcUrl:" + database.getUrl()
+ " you can try again later or reset datasource"); + " you can try again later or reset datasource");
} }
} }
return conn; return conn;
} }
private Connection getConnectionWithRetry(DatabaseResp databaseResp) { private Connection getConnectionWithRetry(Database database) {
int rc = 1; int rc = 1;
for (; ; ) { for (; ; ) {
@@ -176,7 +175,7 @@ public class JdbcDataSourceUtils {
} }
try { try {
Connection connection = getDataSource(databaseResp).getConnection(); Connection connection = getDataSource(database).getConnection();
if (connection != null && connection.isValid(5)) { if (connection != null && connection.isValid(5)) {
return connection; return connection;
} }
@@ -194,7 +193,7 @@ public class JdbcDataSourceUtils {
} }
} }
public void releaseDataSource(DatabaseResp databaseResp) { public void releaseDataSource(Database database) {
jdbcDataSource.removeDatasource(databaseResp); jdbcDataSource.removeDatasource(database);
} }
} }

32
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -5,8 +5,8 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.rmi.ServerException; import java.rmi.ServerException;
import java.sql.Connection; import java.sql.Connection;
@@ -37,7 +37,7 @@ import org.springframework.stereotype.Component;
public class SqlUtils { public class SqlUtils {
@Getter @Getter
private DatabaseResp databaseResp; private Database database;
@Autowired @Autowired
private JdbcDataSource jdbcDataSource; private JdbcDataSource jdbcDataSource;
@@ -58,20 +58,20 @@ public class SqlUtils {
} }
public SqlUtils(DatabaseResp databaseResp) { public SqlUtils(Database database) {
this.databaseResp = databaseResp; this.database = database;
this.dataTypeEnum = DataType.urlOf(databaseResp.getUrl()); this.dataTypeEnum = DataType.urlOf(database.getUrl());
} }
public SqlUtils init(DatabaseResp databaseResp) { public SqlUtils init(Database database) {
//todo Password decryption //todo Password decryption
return SqlUtilsBuilder return SqlUtilsBuilder
.getBuilder() .getBuilder()
.withName(databaseResp.getId() + AT_SYMBOL + databaseResp.getName()) .withName(database.getId() + AT_SYMBOL + database.getName())
.withType(databaseResp.getType()) .withType(database.getType())
.withJdbcUrl(databaseResp.getUrl()) .withJdbcUrl(database.getUrl())
.withUsername(databaseResp.getUsername()) .withUsername(database.getUsername())
.withPassword(databaseResp.getPassword()) .withPassword(database.getPassword())
.withJdbcDataSource(this.jdbcDataSource) .withJdbcDataSource(this.jdbcDataSource)
.withResultLimit(this.resultLimit) .withResultLimit(this.resultLimit)
.withIsQueryLogEnable(this.isQueryLogEnable) .withIsQueryLogEnable(this.isQueryLogEnable)
@@ -96,15 +96,15 @@ public class SqlUtils {
public JdbcTemplate jdbcTemplate() throws RuntimeException { public JdbcTemplate jdbcTemplate() throws RuntimeException {
Connection connection = null; Connection connection = null;
try { try {
connection = jdbcDataSourceUtils.getConnection(databaseResp); connection = jdbcDataSourceUtils.getConnection(database);
} catch (Exception e) { } catch (Exception e) {
log.warn("e:", e); log.warn("e:", e);
} finally { } finally {
JdbcDataSourceUtils.releaseConnection(connection); JdbcDataSourceUtils.releaseConnection(connection);
} }
DataSource dataSource = jdbcDataSourceUtils.getDataSource(databaseResp); DataSource dataSource = jdbcDataSourceUtils.getDataSource(database);
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
jdbcTemplate.setDatabaseProductName(databaseResp.getName()); jdbcTemplate.setDatabaseProductName(database.getName());
jdbcTemplate.setFetchSize(500); jdbcTemplate.setFetchSize(500);
log.info("jdbcTemplate:{}, dataSource:{}", jdbcTemplate, dataSource); log.info("jdbcTemplate:{}, dataSource:{}", jdbcTemplate, dataSource);
return jdbcTemplate; return jdbcTemplate;
@@ -234,7 +234,7 @@ public class SqlUtils {
} }
public SqlUtils build() { public SqlUtils build() {
DatabaseResp databaseResp = DatabaseResp.builder() Database database = Database.builder()
.name(this.name) .name(this.name)
.type(this.type) .type(this.type)
.url(this.jdbcUrl) .url(this.jdbcUrl)
@@ -242,7 +242,7 @@ public class SqlUtils {
.password(this.password) .password(this.password)
.build(); .build();
SqlUtils sqlUtils = new SqlUtils(databaseResp); SqlUtils sqlUtils = new SqlUtils(database);
sqlUtils.jdbcDataSource = this.jdbcDataSource; sqlUtils.jdbcDataSource = this.jdbcDataSource;
sqlUtils.resultLimit = this.resultLimit; sqlUtils.resultLimit = this.resultLimit;
sqlUtils.isQueryLogEnable = this.isQueryLogEnable; sqlUtils.isQueryLogEnable = this.isQueryLogEnable;

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java
View File

@@ -27,6 +27,7 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple; import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -77,7 +78,7 @@ public class HeadlessSchemaManager {
Map<Long, String> modelIdName = new HashMap<>(); Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next()); DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
semanticModel.setDatabaseResp(databaseResp); semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds)); List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) { if (!CollectionUtils.isEmpty(modelRelas)) {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -133,7 +133,7 @@ public class DatabaseServiceImpl implements DatabaseService {
@Override @Override
public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) { public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) {
return queryWithColumns(sql, databaseResp); return queryWithColumns(sql, DatabaseConverter.convert(databaseResp));
} }
@Override @Override
@@ -143,9 +143,9 @@ public class DatabaseServiceImpl implements DatabaseService {
LinkedHashMap::putAll); LinkedHashMap::putAll);
} }
private QueryResultWithSchemaResp queryWithColumns(String sql, DatabaseResp databaseResp) { private QueryResultWithSchemaResp queryWithColumns(String sql, Database database) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp); SqlUtils sqlUtils = this.sqlUtils.init(database);
log.info("query SQL: {}", sql); log.info("query SQL: {}", sql);
sqlUtils.queryInternal(sql, queryResultWithColumns); sqlUtils.queryInternal(sql, queryResultWithColumns);
return queryResultWithColumns; return queryResultWithColumns;
@@ -160,7 +160,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DatabaseResp databaseResp = getDatabase(id); DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl(); String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl();
return queryWithColumns(metaQueryTpl, databaseResp); return queryWithColumns(metaQueryTpl, DatabaseConverter.convert(databaseResp));
} }
@Override @Override
@@ -169,7 +169,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl(); String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db); String metaQuerySql = String.format(metaQueryTpl, db);
return queryWithColumns(metaQuerySql, databaseResp); return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
} }
@Override @Override
@@ -178,7 +178,7 @@ public class DatabaseServiceImpl implements DatabaseService {
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl(); String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();
String metaQuerySql = String.format(metaQueryTpl, db, table); String metaQuerySql = String.format(metaQueryTpl, db, table);
return queryWithColumns(metaQuerySql, databaseResp); return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
} }
} }

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java
View File

@@ -1,16 +1,15 @@
package com.tencent.supersonic.headless.server.service.impl; package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser; import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
@@ -98,10 +97,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception { private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr()); return headlessSchemaManager.get(queryStructReq.getModelIdStr());
ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq);
semanticModel.setDataDate(itemDateResp);
return semanticModel;
} }
} }

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java
View File

@@ -9,8 +9,22 @@ import com.tencent.supersonic.headless.core.pojo.Database;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import java.util.Arrays; import java.util.Arrays;
public class DatabaseConverter { public class DatabaseConverter {
public static Database convert(DatabaseResp databaseResp) {
Database database = new Database();
BeanUtils.copyProperties(databaseResp, database);
ConnectInfo connectInfo = new ConnectInfo();
connectInfo.setUserName(databaseResp.getUsername());
connectInfo.setPassword(databaseResp.getPassword());
connectInfo.setUrl(databaseResp.getUrl());
connectInfo.setDatabase(databaseResp.getDatabase());
database.setConnectInfo(connectInfo);
database.setVersion(databaseResp.getVersion());
return database;
}
public static Database convert(DatabaseReq databaseReq) { public static Database convert(DatabaseReq databaseReq) {
Database database = new Database(); Database database = new Database();
BeanUtils.copyProperties(databaseReq, database); BeanUtils.copyProperties(databaseReq, database);