topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Support optimizing and generating SQL based on the engine type. (#666)

Browse Source
This commit is contained in:
lexluo09
2024-01-19 22:32:28 +08:00
committed by GitHub
parent 20c8456705
commit 9c6bd7cf19
30 changed files with 309 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java
View File

@@ -14,17 +14,11 @@ public class PostgresqlParametersBuilder implements DbParametersBuilder {
public List<DatabaseParameter> build() {
List<DatabaseParameter> databaseParameters = new ArrayList<>();
DatabaseParameter host = new DatabaseParameter();
host.setComment("host");
host.setName("host");
host.setPlaceholder("请输入host");
host.setComment("链接");
host.setName("url");
host.setPlaceholder("请输入链接");
databaseParameters.add(host);
DatabaseParameter port = new DatabaseParameter();
port.setComment("port");
port.setName("port");
port.setPlaceholder("请输入端口号");
databaseParameters.add(port);
DatabaseParameter userName = new DatabaseParameter();
userName.setComment("用户名");
userName.setName("username");

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -60,7 +60,6 @@ public class QueryController {
User user = UserHolder.findUser(request, response);
QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq);
return queryService.queryBySql(querySqlReq, user);
//return queryService.queryByStructWithAuth(queryStructReq, user);
}
@PostMapping("/queryMetricDataById")

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -21,7 +21,7 @@ import java.util.List;
public interface QueryService {
Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
SemanticQueryResp queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

62
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -44,7 +44,6 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.server.cache.QueryCache;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
@@ -67,7 +66,6 @@ import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -79,17 +77,11 @@ public class QueryServiceImpl implements QueryService {
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private final CacheManager cacheManager;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final Catalog catalog;
private final AppService appService;
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
private final QueryCache queryCache;
private final SemanticSchemaManager semanticSchemaManager;
private final QueryParser queryParser;
@@ -98,7 +90,6 @@ public class QueryServiceImpl implements QueryService {
public QueryServiceImpl(
StatUtils statUtils,
CacheManager cacheManager,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
Catalog catalog,
@@ -108,7 +99,6 @@ public class QueryServiceImpl implements QueryService {
QueryParser queryParser,
QueryPlanner queryPlanner) {
this.statUtils = statUtils;
this.cacheManager = cacheManager;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.catalog = catalog;
@@ -122,10 +112,7 @@ public class QueryServiceImpl implements QueryService {
@Override
@S2SQLDataPermission
@SneakyThrows
public Object queryBySql(QuerySqlReq querySQLReq, User user) {
statUtils.initStatInfo(querySQLReq, user);
QueryStatement queryStatement = new QueryStatement();
SemanticQueryResp results = null;
public SemanticQueryResp queryBySql(QuerySqlReq querySQLReq, User user) {
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
try {
//1.initStatInfo
@@ -137,23 +124,25 @@ public class QueryServiceImpl implements QueryService {
}
StatUtils.get().setUseResultCache(false);
//3 query from db
queryStatement = convertToQueryStatement(querySQLReq, user);
QueryStatement queryStatement = convertToQueryStatement(querySQLReq, user);
log.info("queryStatement:{}", queryStatement);
results = query(queryStatement);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(querySQLReq, results);
Boolean setCacheSuccess = queryCache.put(querySQLReq, result);
if (setCacheSuccess) {
// if semanticQueryResp is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(querySQLReq));
}
if (Objects.isNull(results)) {
if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR;
}
return result;
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:", e);
throw e;
} finally {
statUtils.statInfo2DbAsync(state);
}
statUtils.statInfo2DbAsync(state);
return results;
}
public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) {
@@ -186,7 +175,6 @@ public class QueryServiceImpl implements QueryService {
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = null;
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
log.info("[queryStructReq:{}]", queryStructReq);
try {
@@ -200,17 +188,17 @@ public class QueryServiceImpl implements QueryService {
StatUtils.get().setUseResultCache(false);
//3 query
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
semanticQueryResp = query(queryStatement);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
Boolean setCacheSuccess = queryCache.put(queryStructReq, result);
if (setCacheSuccess) {
// if semanticQueryResp is not null, update cache data
// if result is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
}
if (Objects.isNull(semanticQueryResp)) {
if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR;
}
return semanticQueryResp;
return result;
} catch (Exception e) {
log.error("exception in queryByStruct, e: ", e);
state = TaskStatusEnum.ERROR;
@@ -447,28 +435,6 @@ public class QueryServiceImpl implements QueryService {
return ExplainResp.builder().sql(sql).build();
}
private boolean isCache(QueryMultiStructReq queryStructReq) {
if (!cacheEnable) {
return false;
}
if (!CollectionUtils.isEmpty(queryStructReq.getQueryStructReqs())
&& queryStructReq.getQueryStructReqs().get(0).getCacheInfo() != null) {
return queryStructReq.getQueryStructReqs().get(0).getCacheInfo().getCache();
}
return false;
}
private SemanticQueryResp queryByCache(String key, Object queryCmd) {
Object resultObject = cacheManager.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
statUtils.updateResultCacheKey(key);
return (SemanticQueryResp) resultObject;
}
return null;
}
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -115,7 +115,7 @@ public class QueryReqConverter {
result.setRootPath(querySQLReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()),
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
@@ -43,7 +44,8 @@ class HeadlessParserServiceTest {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricReq(metricCommand);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
sqlParser.setSql(aggBuilder.getSql());
EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType));
sqlParser.setSourceId(aggBuilder.getSourceId());
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());