topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Headless] Simplify the QueryService interface, optimize Query permissions, and add integration testing. (#687)

Browse Source
This commit is contained in:
lexluo09
2024-01-24 17:33:12 +08:00
committed by GitHub
parent 48fb01f6bc
commit 922201c181
29 changed files with 529 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2SQLDataPermission.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java
View File

@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface S2SQLDataPermission {
public @interface S2DataPermission {
}

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/StructDataPermission.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.headless.server.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.PARAMETER, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface StructDataPermission {
}

137
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -4,17 +4,26 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -26,14 +35,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Aspect
@Component
@Slf4j
@@ -41,51 +42,79 @@ public class DimValueAspect {
@Value("${dimension.value.map.enable:true}")
private Boolean dimensionValueMapEnable;
@Value("${dimension.value.map.sql.enable:true}")
private Boolean dimensionValueMapSqlEnable;
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.headless.server.service.impl.QueryServiceImpl.queryBySql(..))")
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapSqlEnable) {
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
@Around("execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByReq(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
SemanticQueryReq queryReq = (SemanticQueryReq) args[0];
if (queryReq instanceof QueryStructReq) {
return handleStructDimValue(joinPoint);
}
if (queryReq instanceof QuerySqlReq) {
return handleSqlDimValue(joinPoint);
}
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
}
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds()));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
SemanticQueryResp semanticQueryResp = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(semanticQueryResp)) {
rewriteDimValue(semanticQueryResp, dimAndTechNameAndBizNamePair);
}
return semanticQueryResp;
}
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs();
QuerySqlReq querySQLReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds()));
String sql = querySQLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
fieldExpressionList.stream().forEach(expression -> {
if (fieldNames.contains(expression.getFieldName())) {
dimensions.stream().forEach(dimension -> {
if (expression.getFieldName().equals(dimension.getName())
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
// consider '=' filter
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(),
dimValue.getTechName(), expression.getFieldName());
}
});
}
// consider 'in' filter,each element needs to judge.
replaceInCondition(expression, dimension, filedNameToValueMap);
}
});
for (FieldExpression expression : fieldExpressionList) {
if (!fieldNames.contains(expression.getFieldName())) {
continue;
}
});
log.info("filedNameToValueMap:{}", filedNameToValueMap);
for (DimensionResp dimension : dimensions) {
if (!expression.getFieldName().equals(dimension.getName())
|| CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
continue;
}
// consider '=' filter
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(),
dimValue.getTechName(), expression.getFieldName());
}
});
}
// consider 'in' filter,each element needs to judge.
replaceInCondition(expression, dimension, filedNameToValueMap);
}
}
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
querySQLReq.setSql(sql);
@@ -99,7 +128,7 @@ public class DimValueAspect {
}
public void replaceInCondition(FieldExpression expression, DimensionResp dimension,
Map<String, Map<String, String>> filedNameToValueMap) {
Map<String, Map<String, String>> filedNameToValueMap) {
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
fieldValue = fieldValue.replace("'", "");
@@ -127,40 +156,12 @@ public class DimValueAspect {
}
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
String oldValue, String newValue, String fieldName) {
String oldValue, String newValue, String fieldName) {
Map<String, String> map = new HashMap<>();
map.put(oldValue, newValue);
filedNameToValueMap.put(fieldName, map);
}
@Around("execution(* com.tencent.supersonic.headless.server.rest.QueryController.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStructWithAuth(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds()));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
SemanticQueryResp semanticQueryResp = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(semanticQueryResp)) {
rewriteDimValue(semanticQueryResp, dimAndTechNameAndBizNamePair);
}
return semanticQueryResp;
}
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {

221
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -7,9 +7,14 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
@@ -21,6 +26,7 @@ import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
@@ -44,7 +50,7 @@ import org.springframework.util.CollectionUtils;
@Aspect
@Order(1)
@Slf4j
public class S2SQLDataAspect extends AuthCheckBaseAspect {
public class S2DataPermissionAspect extends AuthCheckBaseAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@@ -55,33 +61,50 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission)")
private void s2SQLPermissionCheck() {
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
private void s2PermissionCheck() {
}
@Around("s2SQLPermissionCheck()")
@Around("s2PermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2SQL permission check!");
Object[] objects = joinPoint.getArgs();
QuerySqlReq querySQLReq = (QuerySqlReq) objects[0];
User user = (User) objects[1];
log.info("s2 permission check!");
if (!permissionDataEnable) {
log.info("not to check s2SQL permission!");
log.info("not to check permission!");
return joinPoint.proceed();
}
Object[] objects = joinPoint.getArgs();
SemanticQueryReq queryReq = (SemanticQueryReq) objects[0];
if (!queryReq.isNeedAuth()) {
log.info("needAuth is false, there is no need to check permissions.");
return joinPoint.proceed();
}
User user = (User) objects[1];
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = querySQLReq.getModelIds();
//1. determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
log.info("determine whether admin of the model!");
// determine whether admin of the model
if (doModelAdmin(user, queryReq.getModelIds())) {
return joinPoint.proceed();
}
// 2. determine whether the subject field is visible
doModelVisible(user, modelIds);
// 3. fetch data permission meta information
// determine whether the subject field is visible
doModelVisible(user, queryReq.getModelIds());
if (queryReq instanceof QuerySqlReq) {
return checkSqlPermission(joinPoint, (QuerySqlReq) queryReq);
}
if (queryReq instanceof QueryStructReq) {
return checkStructPermission(joinPoint, (QueryStructReq) queryReq);
}
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
}
private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySQLReq)
throws Throwable {
Object[] objects = joinPoint.getArgs();
User user = (User) objects[1];
List<Long> modelIds = querySQLReq.getModelIds();
// fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
@@ -95,13 +118,13 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
// row permission pre-filter
doRowPermission(querySQLReq, authorizedResource);
// 6.proceed
// proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
@@ -110,7 +133,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
}
// 6.if the column has no permission, hit *
// if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
@@ -121,6 +144,111 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
return queryResultAfterDesensitization;
}
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
MetaFilter metaFilter = new MetaFilter(modelIds);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
ModelResp modelResp = modelInfos.get(0);
List<String> admins = modelService.getModelAdmin(modelResp.getId());
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
public Object checkStructPermission(ProceedingJoinPoint point, QueryStructReq queryStructReq) throws Throwable {
Object[] args = point.getArgs();
User user = (User) args[1];
// fetch data permission meta information
List<Long> modelIds = queryStructReq.getModelIds();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIds, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource,
queryStructReq.getModelIds());
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
// row permission pre-filter
doRowPermission(queryStructReq, authorizedResource);
// proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
}
// if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
}
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
return false;
}
private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
@@ -154,33 +282,36 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
}
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
authorizedResource.getFilters().stream()
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
}
MetaFilter metaFilter = new MetaFilter(modelIds);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
ModelResp modelResp = modelInfos.get(0);
List<String> admins = modelService.getModelAdmin(modelResp.getId());
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
if (CollectionUtils.isEmpty(dimensionFilters)) {
log.debug("dimensionFilters is empty");
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
if (StringUtils.isNotEmpty(joiner.toString())) {
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
filters.add(filter);
queryStructReq.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);
}
}
}

183
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java
View File

@@ -1,183 +0,0 @@
package com.tencent.supersonic.headless.server.aspect;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Slf4j
public class StructDataAspect extends AuthCheckBaseAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private DimensionService dimensionService;
@Autowired
private ModelService modelService;
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.StructDataPermission)")
public void dataPermissionAOP() {
}
@Around(value = "dataPermissionAOP()")
public Object around(ProceedingJoinPoint point) throws Throwable {
Object[] args = point.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
User user = (User) args[1];
if (!permissionDataEnable) {
log.info("permissionDataEnable is false");
return point.proceed();
}
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("lease provide user information");
}
//1. determine whether admin of the model
if (doModelAdmin(user, queryStructReq.getModelIds())) {
return point.proceed();
}
// 2. determine whether the subject field is visible
doModelVisible(user, queryStructReq.getModelIds());
// 3. fetch data permission meta information
List<Long> modelIds = queryStructReq.getModelIds();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIds, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource,
queryStructReq.getModelIds());
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
doRowPermission(queryStructReq, authorizedResource);
// 6.proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
}
// 6.if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
}
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
return false;
}
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
authorizedResource.getFilters().stream()
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
}
if (CollectionUtils.isEmpty(dimensionFilters)) {
log.debug("dimensionFilters is empty");
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
if (StringUtils.isNotEmpty(joiner.toString())) {
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
filters.add(filter);
queryStructReq.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);
}
}
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
}

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -8,7 +8,6 @@ import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
@@ -18,23 +17,19 @@ import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.QueryService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
@@ -51,7 +46,7 @@ public class QueryController {
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryBySql(querySQLReq, user);
return queryService.queryByReq(querySQLReq, user);
}
@PostMapping("/struct")
@@ -60,7 +55,7 @@ public class QueryController {
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq, true);
return queryService.queryBySql(querySqlReq, user);
return queryService.queryByReq(querySqlReq, user);
}
@PostMapping("/queryMetricDataById")
@@ -85,19 +80,6 @@ public class QueryController {
downloadService.batchDownload(batchDownloadReq, user, response);
}
@PostMapping("/queryStatement")
public SemanticQueryResp queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
return queryService.queryByQueryStatement(queryStatement);
}
@PostMapping("/struct/parse")
public SqlParserResp parseByStruct(@RequestBody ParseSqlReq parseSqlReq) throws Exception {
QueryStatement queryStatement = queryService.explain(parseSqlReq);
SqlParserResp sqlParserResp = new SqlParserResp();
BeanUtils.copyProperties(queryStatement, sqlParserResp);
return sqlParserResp;
}
/**
* queryByMultiStruct
*/
@@ -106,7 +88,7 @@ public class QueryController {
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByMultiStruct(queryMultiStructReq, user);
return queryService.queryByReq(queryMultiStructReq, user);
}
/**
@@ -132,23 +114,19 @@ public class QueryController {
public <T> ExplainResp explain(@RequestBody ExplainSqlReq<T> explainSqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq());
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryType.SQL.equals(queryTypeEnum)) {
QuerySqlReq querySQLReq = JsonUtil.toObject(queryReqJson, QuerySqlReq.class);
if (QueryType.SQL.equals(explainSqlReq.getQueryTypeEnum())) {
ExplainSqlReq<QuerySqlReq> explainSqlReqNew = ExplainSqlReq.<QuerySqlReq>builder()
.queryReq(querySQLReq)
.queryTypeEnum(queryTypeEnum).build();
.queryReq(JsonUtil.toObject(queryReqJson, QuerySqlReq.class))
.queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build();
return queryService.explain(explainSqlReqNew, user);
}
if (QueryType.STRUCT.equals(queryTypeEnum)) {
QueryStructReq queryStructReq = JsonUtil.toObject(queryReqJson, QueryStructReq.class);
if (QueryType.STRUCT.equals(explainSqlReq.getQueryTypeEnum())) {
ExplainSqlReq<QueryStructReq> explainSqlReqNew = ExplainSqlReq.<QueryStructReq>builder()
.queryReq(queryStructReq)
.queryTypeEnum(queryTypeEnum).build();
.queryReq(JsonUtil.toObject(queryReqJson, QueryStructReq.class))
.queryTypeEnum(explainSqlReq.getQueryTypeEnum()).build();
return queryService.explain(explainSqlReqNew, user);
}
return null;

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -3,46 +3,28 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
public interface QueryService {
SemanticQueryResp queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
SemanticQueryResp queryBySemanticQuery(SemanticQueryReq semanticQueryReq, User user) throws Exception;
SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception;
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
QueryStatement explain(ParseSqlReq parseSqlReq) throws Exception;
@ApiHeaderCheck
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq,
HttpServletRequest request) throws Exception;
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -74,7 +74,7 @@ public class DownloadServiceImpl implements DownloadService {
File file = FileUtils.createTmpFile(fileName);
try {
QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true);
SemanticQueryResp queryResult = (SemanticQueryResp) queryService.queryBySql(querySqlReq, user);
SemanticQueryResp queryResult = (SemanticQueryResp) queryService.queryByReq(querySqlReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
} catch (RuntimeException e) {
@@ -114,7 +114,7 @@ public class DownloadServiceImpl implements DownloadService {
for (MetricSchemaResp metric : metrics) {
try {
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
SemanticQueryResp queryResult = queryService.queryByReq(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders()).build();

170
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -20,7 +20,6 @@ import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
@@ -43,8 +42,7 @@ import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.planner.QueryPlanner;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
@@ -56,7 +54,6 @@ import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -111,30 +108,40 @@ public class QueryServiceImpl implements QueryService {
}
@Override
@S2SQLDataPermission
@S2DataPermission
@SneakyThrows
public SemanticQueryResp queryBySql(QuerySqlReq querySQLReq, User user) {
return queryBySemanticQuery(querySQLReq, user);
}
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception {
return queryBySemanticQuery(queryStructCmd, user);
}
public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) {
SemanticQueryResp queryResultWithColumns = null;
QueryExecutor queryExecutor = queryPlanner.route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(queryStatement);
queryResultWithColumns.setSql(queryStatement.getSql());
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds());
public SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) {
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
log.info("[queryReq:{}]", queryReq);
try {
//1.initStatInfo
statUtils.initStatInfo(queryReq, user);
//2.query from cache
Object query = queryCache.query(queryReq);
if (Objects.nonNull(query)) {
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
//3 query
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(queryReq, result);
if (setCacheSuccess) {
// if result is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryReq));
}
if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR;
}
return result;
} catch (Exception e) {
log.error("exception in queryByStruct, e: ", e);
state = TaskStatusEnum.ERROR;
throw e;
} finally {
statUtils.statInfo2DbAsync(state);
}
return queryResultWithColumns;
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception {
@@ -150,41 +157,6 @@ public class QueryServiceImpl implements QueryService {
return queryStatement;
}
@Override
public SemanticQueryResp queryBySemanticQuery(SemanticQueryReq semanticQueryReq, User user) throws Exception {
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
log.info("[semanticQueryReq:{}]", semanticQueryReq);
try {
//1.initStatInfo
statUtils.initStatInfo(semanticQueryReq, user);
//2.query from cache
Object query = queryCache.query(semanticQueryReq);
if (Objects.nonNull(query)) {
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
//3 query
QueryStatement queryStatement = buildQueryStatement(semanticQueryReq, user);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(semanticQueryReq, result);
if (setCacheSuccess) {
// if result is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(semanticQueryReq));
}
if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR;
}
return result;
} catch (Exception e) {
log.error("exception in queryByStruct, e: ", e);
state = TaskStatusEnum.ERROR;
throw e;
} finally {
statUtils.statInfo2DbAsync(state);
}
}
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception {
if (semanticQueryReq instanceof QuerySqlReq) {
return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user);
@@ -225,66 +197,11 @@ public class QueryServiceImpl implements QueryService {
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
}
@Override
@StructDataPermission
@SneakyThrows
public SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) {
return queryByStruct(queryStructReq, user);
}
@Override
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
throws Exception {
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
try {
//1.initStatInfo
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
//2.query from cache
Object query = queryCache.query(queryMultiStructReq);
if (Objects.nonNull(query)) {
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
//3.parse and optimizer
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
queryParser.parse(queryStatement);
queryPlanner.plan(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
QueryStatement queryStatement = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
//4.route
QueryExecutor executor = queryPlanner.route(queryStatement);
SemanticQueryResp semanticQueryResp = null;
if (executor != null) {
semanticQueryResp = executor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
if (Objects.isNull(semanticQueryResp)) {
state = TaskStatusEnum.ERROR;
}
return semanticQueryResp;
} catch (Exception e) {
log.error("exception in queryByMultiStruct, e: ", e);
state = TaskStatusEnum.ERROR;
throw e;
} finally {
statUtils.statInfo2DbAsync(state);
}
}
@Override
@SneakyThrows
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq);
return queryBySql(querySQLReq, user);
return queryByReq(querySQLReq, user);
}
@Override
@@ -308,22 +225,6 @@ public class QueryServiceImpl implements QueryService {
return getExplainResp(queryStatement);
}
@Override
public QueryStatement explain(ParseSqlReq parseSqlReq) throws Exception {
QueryStructReq queryStructCmd = new QueryStructReq();
Set<Long> models = new HashSet<>();
models.add(Long.valueOf(parseSqlReq.getRootPath()));
queryStructCmd.setModelIds(models);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setParseSqlReq(parseSqlReq);
queryStatement.setSql(parseSqlReq.getSql());
queryStatement.setIsS2SQL(true);
SemanticModel semanticModel = semanticSchemaManager.get(parseSqlReq.getRootPath());
queryStatement.setSemanticModel(semanticModel);
return plan(queryStatement);
}
@Override
public ItemQueryResultResp queryMetricDataById(QueryItemReq queryItemReq,
HttpServletRequest request) throws Exception {
@@ -348,15 +249,14 @@ public class QueryServiceImpl implements QueryService {
item.setName(metricResp.getName());
List<Item> items = item.getRelateItems();
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!org.springframework.util.CollectionUtils.isEmpty(items)) {
if (!CollectionUtils.isEmpty(items)) {
List<Long> ids = items.stream().map(Item::getId).collect(Collectors.toList());
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setIds(ids);
dimensionResps = catalog.getDimensions(dimensionFilter);
}
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit);
SemanticQueryResp semanticQueryResp =
queryByStruct(queryStructReq, User.getAppUser(appId));
SemanticQueryResp semanticQueryResp = queryByReq(queryStructReq, User.getAppUser(appId));
SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult();
apiQuerySingleResult.setItem(item);
apiQuerySingleResult.setResult(semanticQueryResp);

2
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java
View File

@@ -32,7 +32,7 @@ class DownloadServiceImplTest {
ModelService modelService = Mockito.mock(ModelService.class);
QueryService queryService = Mockito.mock(QueryService.class);
when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp()));
when(queryService.queryByStruct(any(), any())).thenReturn(mockQueryResult());
when(queryService.queryByReq(any(), any())).thenReturn(mockQueryResult());
DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService);
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
File file = FileUtils.createTmpFile(fileName);