topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Headless] Fix the caching issue in struct queries and add headless test cases. (#689)

Browse Source
This commit is contained in:
lexluo09
2024-01-25 11:31:15 +08:00
committed by GitHub
parent 922201c181
commit 841db25198
23 changed files with 214 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -272,7 +272,8 @@ public class QueryStructReq extends SemanticQueryReq {
}
public String getModelName() {
return Objects.nonNull(modelName) ? modelName : "m_" + String.valueOf(StringUtils.join(modelIds, "_"));
return Objects.nonNull(modelName) ? modelName :
Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_");
}
}

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
View File

@@ -19,13 +19,12 @@ public abstract class SemanticQueryReq {
protected boolean needAuth = true;
protected Set<Long> modelIds;
protected Set<Long> modelIds = new HashSet<>();
protected List<Param> params = new ArrayList<>();
protected Cache cacheInfo = new Cache();
public void setModelId(Long modelId) {
modelIds = new HashSet<>();
public void addModelId(Long modelId) {
modelIds.add(modelId);
}

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java vendored
View File

@@ -20,8 +20,7 @@ public class DefaultQueryCache implements QueryCache {
@Autowired
private CacheManager cacheManager;
public Object query(SemanticQueryReq semanticQueryReq) {
String cacheKey = getCacheKey(semanticQueryReq);
public Object query(SemanticQueryReq semanticQueryReq, String cacheKey) {
if (isCache(semanticQueryReq)) {
Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
@@ -30,15 +29,14 @@ public class DefaultQueryCache implements QueryCache {
return null;
}
public Boolean put(SemanticQueryReq semanticQueryReq, Object value) {
public Boolean put(String cacheKey, Object value) {
if (cacheEnable && Objects.nonNull(value)) {
String key = getCacheKey(semanticQueryReq);
CompletableFuture.supplyAsync(() -> cacheManager.put(key, value))
CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
log.info("add record to cache, key:{}", key);
log.info("add record to cache, key:{}", cacheKey);
return true;
}
return false;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java vendored
View File

@@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
public interface QueryCache {
Object query(SemanticQueryReq semanticQueryReq);
Object query(SemanticQueryReq semanticQueryReq, String cacheKey);
Boolean put(SemanticQueryReq semanticQueryReq, Object value);
Boolean put(String cacheKey, Object value);
String getCacheKey(SemanticQueryReq semanticQueryReq);

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -84,9 +84,9 @@ public class DimValueAspect {
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs();
QuerySqlReq querySQLReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds()));
String sql = querySQLReq.getSql();
QuerySqlReq querySqlReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySqlReq.getModelIds()));
String sql = querySqlReq.getSql();
log.info("correctorSql before replacing:{}", sql);
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
@@ -117,7 +117,7 @@ public class DimValueAspect {
}
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
querySQLReq.setSql(sql);
querySqlReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -99,13 +99,13 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
}
private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySQLReq)
private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq)
throws Throwable {
Object[] objects = joinPoint.getArgs();
User user = (User) objects[1];
List<Long> modelIds = querySQLReq.getModelIds();
List<Long> modelIds = querySqlReq.getModelIds();
// fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, user);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
@@ -119,10 +119,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
// row permission pre-filter
doRowPermission(querySQLReq, authorizedResource);
doRowPermission(querySqlReq, authorizedResource);
// proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
@@ -144,14 +144,14 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
return queryResultAfterDesensitization;
}
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName,
private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq);
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
@@ -249,7 +249,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
return false;
}
private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -271,10 +271,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
querySQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
String sql = SqlParserAddHelper.addWhere(querySqlReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
querySqlReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -42,11 +42,11 @@ public class QueryController {
private DownloadService downloadService;
@PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySQLReq,
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByReq(querySQLReq, user);
return queryService.queryByReq(querySqlReq, user);
}
@PostMapping("/struct")

29
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -117,7 +117,8 @@ public class QueryServiceImpl implements QueryService {
//1.initStatInfo
statUtils.initStatInfo(queryReq, user);
//2.query from cache
Object query = queryCache.query(queryReq);
String cacheKey = queryCache.getCacheKey(queryReq);
Object query = queryCache.query(queryReq, cacheKey);
if (Objects.nonNull(query)) {
return (SemanticQueryResp) query;
}
@@ -126,10 +127,10 @@ public class QueryServiceImpl implements QueryService {
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(queryReq, result);
Boolean setCacheSuccess = queryCache.put(cacheKey, result);
if (setCacheSuccess) {
// if result is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryReq));
statUtils.updateResultCacheKey(cacheKey);
}
if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR;
@@ -144,15 +145,15 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception {
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(querySQLReq.getModelIds());
filter.setModelIds(querySqlReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps);
queryStatement.setModelIds(querySQLReq.getModelIds());
QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, modelSchemaResps);
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
SemanticModel semanticModel = semanticSchemaManager.get(querySQLReq.getModelIdStr());
SemanticModel semanticModel = semanticSchemaManager.get(querySqlReq.getModelIdStr());
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
@@ -200,8 +201,8 @@ public class QueryServiceImpl implements QueryService {
@Override
@SneakyThrows
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq);
return queryByReq(querySQLReq, user);
QuerySqlReq querySqlReq = buildQuerySqlReq(queryDimValueReq);
return queryByReq(querySqlReq, user);
}
@Override
@@ -312,7 +313,7 @@ public class QueryServiceImpl implements QueryService {
}
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq();
QuerySqlReq querySqlReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId());
@@ -324,9 +325,9 @@ public class QueryServiceImpl implements QueryService {
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
}
querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySQLReq.setSql(sql);
return querySQLReq;
querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySqlReq.setSql(sql);
return querySqlReq;
}
private QueryStatement plan(QueryStatement queryStatement) throws Exception {

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -62,7 +62,7 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QuerySqlReq querySQLReq,
public QueryStatement convert(QuerySqlReq querySqlReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
@@ -71,18 +71,18 @@ public class QueryReqConverter {
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName
convertNameToBizName(querySQLReq, modelSchemaResps);
convertNameToBizName(querySqlReq, modelSchemaResps);
//2.functionName corrector
functionNameCorrector(querySQLReq);
functionNameCorrector(querySqlReq);
//3.correct tableName
correctTableName(querySQLReq);
correctTableName(querySqlReq);
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(querySqlReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
@@ -96,7 +96,7 @@ public class QueryReqConverter {
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : querySQLReq.getModelIds()) {
for (Long modelId : querySqlReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
}
@@ -105,27 +105,27 @@ public class QueryReqConverter {
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList()));
}
AggOption aggOption = getAggOption(querySQLReq);
AggOption aggOption = getAggOption(querySqlReq);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(querySQLReq, result);
BeanUtils.copyProperties(querySqlReq, result);
result.setRootPath(querySQLReq.getModelIdStr());
result.setRootPath(querySqlReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0));
DatabaseResp database = catalog.getDatabaseByModelId(querySqlReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//5. do deriveMetric
generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, aggOption, result);
generateDerivedMetric(querySqlReq.getModelIds(), modelSchemaResps, aggOption, result);
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds()));
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySqlReq.getModelIds()));
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement();
@@ -133,7 +133,7 @@ public class QueryReqConverter {
queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(querySQLReq.getModelIds());
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableLimitWrapper(limitWrapper);
return queryStatement;

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -129,8 +129,8 @@ public class QueryStructUtils {
return resNameEnSet;
}
public Set<String> getResName(QuerySqlReq querySQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql())
public Set<String> getResName(QuerySqlReq querySqlReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySqlReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -140,11 +140,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) {
Set<String> resNameSet = getResName(querySQLReq);
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySqlReq, User user) {
Set<String> resNameSet = getResName(querySqlReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -175,8 +175,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) {
String sql = querySQLReq.getSql();
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
@@ -93,13 +94,17 @@ public class StatUtils {
if (semanticQueryReq instanceof QueryStructReq) {
initStructStatInfo((QueryStructReq) semanticQueryReq, facadeUser);
}
if (semanticQueryReq instanceof QueryMultiStructReq) {
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
initStructStatInfo(queryStructCmd, facadeUser);
}
}
public void initSqlStatInfo(QuerySqlReq querySQLReq, User facadeUser) {
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
queryStatInfo.setModelId(querySQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0));
List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
queryStatInfo.setModelId(querySqlReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySqlReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>();
List<String> metrics = new ArrayList<>();
@@ -111,12 +116,12 @@ public class StatUtils {
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(querySQLReq.getModelIds().get(0))
.setModelId(querySqlReq.getModelIds().get(0))
.setUser(userName)
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(querySQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString()))
.setQuerySqlCmd(querySqlReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(querySqlReq.toString()))
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)